view release on metacpan or  search on metacpan

lib/App/Glacier/Command/Get.pm  view on Meta::CPAN

	
	if ($job->is_completed) {
	    print "completed on ",
	          $job->get('CompletionDate')->canned_format('full-iso'),"\n";
	}
	exit(0);
    }
    
    if ($job->is_completed) {
	my $cache_file = $job->cache_file;
	if (-f $cache_file) {
	    $self->debug(1, "$job: copying from $cache_file");
	    return if $self->dry_run;
	    unless (copy($cache_file, $localname)) {
		$self->abend(EX_FAILURE,
			     "can't copy $cache_file to $localname: $!");
	    }
	} else {
	    my $tree_hash = $self->download($job, $localname);
	    if (!$self->dry_run
		&& $tree_hash ne $job->get('ArchiveSHA256TreeHash')) {
		unlink $localname;
		$self->abend(EX_SOFTWARE, "downloaded file is corrupt");
	    }
	}
    } else {
	my ($status, $message) = $job->status;
	if ($status eq 'InProgress') {
	    $self->abend(EX_TEMPFAIL,
			 "archive retrieval job for $vaultname:$filespec initiated at " .
			 $job->get('CreationDate')->canned_format
			 . "; please retry later to download the file");
	} else {  
	    $self->error("archive retrieval job for $vaultname:$filespec: $status: $message");
	    $self->error("deleting job", $job->id);
	    $job->delete;
	    exit (EX_FAILURE);
	}
    }
}

use constant MB => 1024*1024;
use constant TWOMB => 2*MB;

sub download {
    my ($self, $job, $localname) = @_;
    
    my $archive_size = $job->get('ArchiveSizeInBytes');
    if ($archive_size < $self->cf_transfer_param(qw(download single-part-size))) {
	# simple download 
	$self->_download_simple($job, $localname);
    } else {
	$self->_download_multipart($job, $localname);
    }
}

sub _open_output {
    my ($self, $localname) = @_;
    open(my $fd, '>', $localname)
	or $self->abort(EX_FAILURE, "can't open $localname: $!");
    binmode($fd);
    truncate($fd, 0);
    return $fd;
}

sub _download_simple {
    my ($self, $job, $localname) = @_;

    $self->debug(1, "$job: downloading in single part");
    return if $self->dry_run;
    my $fd = $self->_open_output($localname);
    my ($res, $tree_hash) = $self->glacier->Get_job_output($job->vault,
							   $job->id);
    if ($self->glacier->lasterr) {
	$self->abend(EX_FAILURE, "downoad failed: ",
		     $self->glacier->last_error_message);
    }
    syswrite($fd, $res);
    close($fd);
    return $tree_hash;
}

sub _download_multipart {
    my ($self, $job, $localname) = @_;
        
    my $glacier = $self->{_glacier};

    my $tree_hash;
    
    my $njobs = $self->{_options}{jobs}
                || $self->cf_transfer_param(qw(download jobs));

    my $archive_size = $job->get('ArchiveSizeInBytes');
    my $part_size;
    # Compute approximate part size
    $part_size = ($archive_size - 1) / 10000;
    if ($part_size < TWOMB) {
	$part_size = TWOMB;
    } else {
	# Make sure the chunk is Tree-Hash aligned
	# http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations-range.html?shortFooter=true#checksum-calculations-upload-archive-with-ranges
	$part_size = TWOMB * 2 ** int(log($part_size / TWOMB) / log(2) + 1);
    }
    # Number of parts to download:
    my $total_parts = int(($archive_size + $part_size - 1) / $part_size);
    # Compute the number of parts per job
    my $job_parts = int(($total_parts + $njobs - 1) / $njobs);

    $self->debug(1, "$job: downloading in chunks of $part_size bytes, in $njobs jobs, with $job_parts parts per job");

    return if $self->dry_run;

    use Fcntl qw(SEEK_SET);

    my $fd = $self->_open_output($localname);
    my @part_hashes :shared = ();
    my $p = new App::Glacier::Progress($total_parts,
				       prefix => $localname)
	unless $self->{_options}{quiet};
    for (my $i = 0; $i < $njobs; $i++) {
	my ($thr) = threads->create(