view release on metacpan or  search on metacpan

examples/backup_cpan.pl  view on Meta::CPAN

#!/home/acme/bin/perl
use strict;
use warnings;
use lib 'lib';
use Data::Stream::Bulk::Path::Class;
use Net::Amazon::S3;
use Perl6::Say;
use Path::Class;
use Set::Object;
use Term::ProgressBar::Simple;
use List::Util qw(sum);
use Digest::MD5::File qw(file_md5_hex);
use BerkeleyDB::Manager;
use Cwd;
use Config;

my $m = BerkeleyDB::Manager->new(
    home     => Path::Class::Dir->new(cwd),
    db_class => 'BerkeleyDB::Hash',
    create   => 1,
);
my $db = $m->open_db( file => 'md5_cache' );

my $s3 = Net::Amazon::S3->new(
    aws_access_key_id     => 'XXX',
    aws_secret_access_key => 'XXX',
    retry                 => 1,
);

my $client = Net::Amazon::S3::Client->new( s3 => $s3 );
my $bucket = $client->bucket( name => 'minicpan' );

my $root = '/home/acme/Public/minicpan/';

my $file_stream = Data::Stream::Bulk::Path::Class->new(
    dir        => Path::Class::Dir->new($root),
    only_files => 1,
);

my %files;
my $file_set = Set::Object->new();
until ( $file_stream->is_done ) {
    foreach my $filename ( $file_stream->items ) {
        my $key = $filename->relative($root)->stringify;

        #[rootname]path/to/file.txt:<ctime>,<mtime>,<size>,<inodenum>
        my $stat     = $filename->stat;
        my $ctime    = $stat->ctime;
        my $mtime    = $stat->mtime;
        my $size     = $stat->size;
        my $inodenum = $stat->ino;
        my $cachekey = "$key:$ctime,$mtime,$size,$inodenum";

        $db->db_get( $cachekey, my $md5_hex );
        if ($md5_hex) {

            #say "hit $cachekey $md5hex";
        } else {
            $md5_hex = file_md5_hex($filename)
                || die "Failed to find MD5 for $filename";
            $m->txn_do(
                sub {
                    $db->db_put( $cachekey, $md5_hex );
                }