Amazon-S3-FastUploader
view release on metacpan or search on metacpan
.gitattributes
.shipit
Changes
ignore.txt
lib/Amazon/S3/FastUploader.pm
lib/Amazon/S3/FastUploader/File.pm
Makefile.PL
MANIFEST This list of files
s3uploader.pl
README
t/00-load.t
t/01-load.t
t/boilerplate.t
t/manifest.t
t/pod-coverage.t
t/pod.t
META.yml Module YAML meta-data (added by MakeMaker)
META.json Module JSON meta-data (added by MakeMaker)
{
"abstract" : "fast uploader to Amazon S3",
"author" : [
"DQNEO <dqneoo@gmail.com>"
],
"dynamic_config" : 1,
"generated_by" : "ExtUtils::MakeMaker version 6.62, CPAN::Meta::Converter version 2.112150",
"license" : [
"perl_5"
],
"meta-spec" : {
"url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec",
---
abstract: 'fast uploader to Amazon S3'
author:
- 'DQNEO <dqneoo@gmail.com>'
build_requires:
ExtUtils::MakeMaker: 0
configure_requires:
ExtUtils::MakeMaker: 0
dynamic_config: 1
generated_by: 'ExtUtils::MakeMaker version 6.62, CPAN::Meta::Converter version 2.112150'
license: perl
meta-spec:
lib/Amazon/S3/FastUploader.pm view on Meta::CPAN
our $VERSION = '0.08';
sub new {
my $class = shift;
my $config = shift;
bless { config => $config }, $class;
}
sub upload {
my $self = shift;
my $local_dir = shift;
my $bucket_name = shift;
my $target_dir = shift;
my $config = $self->config;
my $process = $config->{process};
my $s3 = Amazon::S3->new($config);
lib/Amazon/S3/FastUploader.pm view on Meta::CPAN
bucket => $bucket,
config => $config,
});
push @local_files , $file;
};
chdir $local_dir;
find($callback, '.');
if ($process > 1) {
$self->_upload_parallel(\@local_files, $process);
} else {
$self->_upload_single(\@local_files);
}
}
sub _upload_single {
my $self = shift;
my @files = @{ shift; };
$self->_print("uploading by a single process\n");
my $i = 0;
my $total_num = @files;
for my $file (@files) {
$i++;
$file->upload();
$self->_print("ok $i / $total_num " . $file->from_to . "\n");
}
$self->_print(sprintf("%d files uploaded\n" , $i));
}
sub _upload_parallel {
my $self = shift;
my @files = @{ shift; };
my $max = shift;
$self->_print("uploading by multi processes\n");
my $pm = new Parallel::ForkManager($max);
$pm->run_on_finish(
sub {
my ($pid, $exit_code, $ident) = @_;
if ($exit_code != 0) {
# on Windows 7, I saw sometimes error like below:
#URI/_query.pm did not return a true value at C:/Perl/lib/URI/_generic.pm line 3.
#
#Compilation failed in require at C:/Perl/lib/URI/_server.pm line 2.
lib/Amazon/S3/FastUploader.pm view on Meta::CPAN
}
});
my $i = 0;
my $total_num = @files;
for my $file (@files) {
$i++;
$pm->start and next;
$file->upload();
$self->_print("ok $i / $total_num " . $file->from_to . "\n");
$pm->finish;
$i++;
}
$pm->wait_all_children;
my $count = @files;
$self->_print(sprintf("%d files uploaded\n" , $count));
}
sub _print {
my $self = shift;
return unless $self->config->{verbose};
print @_;
}
=head1 NAME
Amazon::S3::FastUploader - fast uploader to Amazon S3
=head1 SYNOPSIS
By this module, you can upload many files to Amazon S3 at the same time
(in another word, in parallel) .
The module uses Parallel::ForkManager internally.
use Amazon::S3::FastUploader;
my $local_dir = '/path/to/dir/';
my $bucket_name = 'myubcket';
my $remote_dir '/path/to/dir/';
my $uploader = Amazon::S3::FastUploader->new({
aws_access_key_id => 'your_key_id',
aws_secret_access_key => 'your_secre_key',
process => 10, # num of proccesses in parallel
secure => 1, # use SSL
encrypt => 1, # use ServerSide Encryption
retry => 5,
verbose => 1, # print log to stdout
acl_short => 'public-read', # private if ommited
});
$uploader->upload($local_dir, $bucket_name, $remote_dir);
=head1 METHODS
=head2 new
Instaniates a new object.
Requires a hashref
=head2 upload $local_dir $bucket_name $remote_dir
upload recursively $local_dir to $remote_dir
=head1 AUTHOR
DQNEO, C<< <dqneoo at gmail.com> >>
=head1 Github Repository
https://github.com/DQNEO/Amazon-S3-FastUploader
Forks & Pull Requests are wellcome!
=head1 BUGS
Please report any bugs or feature requests to C<bug-amazon-s3-fastuploader at rt.cpan.org>, or through
the web interface at L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=Amazon-S3-FastUploader>. I will be notified, and then you'll
automatically be notified of progress on your bug as I make changes.
=head1 SUPPORT
You can find documentation for this module with the perldoc command.
lib/Amazon/S3/FastUploader/File.pm view on Meta::CPAN
".xls" => "application/vnd.ms-excel",
".xml" => "application/xml",
".xpm" => "image/x-xpixmap",
".xsl" => "application/xml",
".xslt" => "application/xslt+xml",
".yaml" => "text/yaml",
".yml" => "text/yaml",
".zip" => "application/zip",
};
sub upload {
my $self = shift;
my $bucket = $self->bucket;
my $s3 = $self->s3;
my $opt = {};
if ($self->config->{encrypt}) {
$opt->{"x-amz-server-side-encryption"} = 'AES256';
}
lib/Amazon/S3/FastUploader/File.pm view on Meta::CPAN
my $is_success = 0;
while (! $is_success && $count_failed < $max_retry) {
$is_success = $bucket->add_key_filename($self->_remote_key, $self->local_path, $opt)
or do { warn $s3->err . ": " . $s3->errstr . "(" . $self->from_to .")" ; $count_failed++; };
if ($is_success) {
return 1;
}
}
die "upload failed " . $self->from_to;
}
sub from_to {
my $self = shift;
return $self->local_path . " -> " . $self->_remote_key;
}
sub _remote_key {
my $self = shift;
my $local_path = $self->{local_path};
$local_path =~ s|^\./||;
return $self->target_dir . $local_path;
}
=head1 NAME
Amazon::S3::FastUploader - fast uploader to Amazon S3
=head1 SYNOPSIS
This is an internal module of Amazon::S3::FastUploader.
=head1 METHODS
=head2 new
takes a hashref:
local_path: full path to a local file
target_dir: dirname on S3
bucket: a bucket object
config: config option (hashref)
=head2 upload
do uploading
content_type (mime type) is automatically determined by its extension name.
=head2 from_to
return a string wich shows source filename and target filename
=cut
1;
s3uploader.pl view on Meta::CPAN
my $config = {
aws_access_key_id => $ENV{AWS_ACCESS_KEY_ID},
aws_secret_access_key => $ENV{AWS_SECRET_ACCESS_KEY},
secure => $opts{ssl},
encrypt => $opts{encrypt},
retry => 5,
process => $opts{process},
verbose => $opts{verbose},
};
my $uploader = Amazon::S3::FastUploader->new($config);
$uploader->upload($local_dir, $bucket_name, $target_dir);
__END__
=head1 Name
S3 Uploader - a script to upload recursively a directory to Amazon S3
=head1 SYNOPSIS
-v verbose
-p max process (default 1 if ommitted)
-s use ssl
-h help (this)
example:
upload.pl -v -p 10 /path/to/dir s3://bucketname/foo/bar/dir/
=cut
( run in 1.634 second using v1.01-cache-2.11-cpan-b16cb0d3907 )