view release on metacpan or  search on metacpan

lib/Brackup/Backup.pm  view on Meta::CPAN

package Brackup::Backup;
use strict;
use warnings;
use Carp qw(croak);
use Brackup::ChunkIterator;
use Brackup::CompositeChunk;
use Brackup::GPGProcManager;
use Brackup::GPGProcess;
use File::Basename;
use File::Temp qw(tempfile);

sub new {
    my ($class, %opts) = @_;
    my $self = bless {}, $class;

    $self->{root}    = delete $opts{root};     # Brackup::Root
    $self->{target}  = delete $opts{target};   # Brackup::Target
    $self->{dryrun}  = delete $opts{dryrun};   # bool
    $self->{verbose} = delete $opts{verbose};  # bool
    $self->{inventory} = delete $opts{inventory};  # bool
    $self->{savefiles} = delete $opts{savefiles};  # bool
    $self->{zenityprogress} = delete $opts{zenityprogress};  # bool

    $self->{modecounts} = {}; # type -> mode(octal) -> count
    $self->{idcounts}   = {}; # type -> uid/gid -> count

    $self->{_uid_map} = {};   # uid -> username
    $self->{_gid_map} = {};   # gid -> group

    $self->{saved_files} = [];   # list of Brackup::File objects backed up
    $self->{unflushed_files} = [];   # list of Brackup::File objects not in backup_file

    croak("Unknown options: " . join(', ', keys %opts)) if %opts;

    return $self;
}

# returns true (a Brackup::BackupStats object) on success, or dies with error
sub backup {
    my ($self, $backup_file) = @_;

    my $root   = $self->{root};
    my $target = $self->{target};

    my $stats  = Brackup::BackupStats->new;

    my @gpg_rcpts = $self->{root}->gpg_rcpts;

    my $n_kb         = 0.0; # num:  kb of all files in root
    my $n_files      = 0;   # int:  # of files in root
    my $n_kb_done    = 0.0; # num:  kb of files already done with (uploaded or skipped)

    # if we're pre-calculating the amount of data we'll
    # actually need to upload, store it here.
    my $n_files_up   = 0;
    my $n_kb_up      = 0.0;
    my $n_kb_up_need = 0.0; # by default, not calculated/used.

    my $n_files_done = 0;   # int
    my @files;         # Brackup::File objs

    $self->debug("Discovering files in ", $root->path, "...\n");
    $self->report_progress(0, "Discovering files in " . $root->path . "...");
    $root->foreach_file(sub {
        my ($file) = @_;  # a Brackup::File
        push @files, $file;
        $self->record_mode_ids($file);
        $n_files++;
        $n_kb += $file->size / 1024;
    });

    $self->debug("Number of files: $n_files\n");
    $stats->timestamp('File Discovery');
    $stats->set('Number of Files' => $n_files);
    $stats->set('Total File Size' => sprintf('%0.01f MB', $n_kb / 1024));

    # calc needed chunks
    if ($ENV{CALC_NEEDED}) {
        my $fn = 0;
        foreach my $f (@files) {
            $fn++;
            if ($fn % 100 == 0) { warn "$fn / $n_files ...\n"; }
            foreach my $pc ($f->chunks) {
                if ($target->stored_chunk_from_inventory($pc)) {
                    $pc->forget_chunkref;
                    next;
                }
                $n_kb_up_need += $pc->length / 1024;
                $pc->forget_chunkref;
            }
        }
        warn "kb need to upload = $n_kb_up_need\n";
        $stats->timestamp('Calc Needed');
    }


    my $chunk_iterator = Brackup::ChunkIterator->new(@files);
    undef @files;
    $stats->timestamp('Chunk Iterator');

    my $gpg_iter;
    my $gpg_pm;   # gpg ProcessManager
    if (@gpg_rcpts) {
        ($chunk_iterator, $gpg_iter) = $chunk_iterator->mux_into(2);
        $gpg_pm = Brackup::GPGProcManager->new($gpg_iter, $target);
    }

    # begin temp backup_file
    my ($metafh, $meta_filename);
    unless ($self->{dryrun}) {
        ($metafh, $meta_filename) = tempfile(
                                             '.' . basename($backup_file) . 'XXXXX',
                                             DIR => dirname($backup_file),
        );
        if (! @gpg_rcpts) {
            if (eval { require IO::Compress::Gzip }) {
                close $metafh;
                $metafh = IO::Compress::Gzip->new($meta_filename)
                    or die "Cannot open tempfile with IO::Compress::Gzip: $IO::Compress::Gzip::GzipError";
            }
        }
        print $metafh $self->backup_header;
    }

    my $cur_file; # current (last seen) file
    my @stored_chunks;
    my $file_has_shown_status = 0;

    my $merge_under = $root->merge_files_under;
    my $comp_chunk  = undef;

    my $end_file = sub {
        return unless $cur_file;
        if ($merge_under && $comp_chunk) {
            # defer recording to backup_file until CompositeChunk finalization
            $self->add_unflushed_file($cur_file, [ @stored_chunks ]);
        }
        else {
            print $metafh $cur_file->as_rfc822([ @stored_chunks ], $self) if $metafh;
        }
        $self->add_saved_file($cur_file, [ @stored_chunks ]) if $self->{savefiles};
        $n_files_done++;
        $n_kb_done += $cur_file->size / 1024;
        $cur_file = undef;
    };
    my $show_status = sub {
        # use either size of files in normal case, or if we pre-calculated
        # the size-to-upload (by looking in inventory, then we'll show the
        # more accurate percentage)
        my $percdone = 100 * ($n_kb_up_need ?
                              ($n_kb_up / $n_kb_up_need) :
                              ($n_kb_done / $n_kb));
        my $mb_remain = ($n_kb_up_need ?
                         ($n_kb_up_need - $n_kb_up) :
                         ($n_kb - $n_kb_done)) / 1024;

        $self->debug(sprintf("* %-60s %d/%d (%0.02f%%; remain: %0.01f MB)",
                             $cur_file->path, $n_files_done, $n_files, $percdone,
                             $mb_remain));

        $self->report_progress($percdone);
    };
    my $start_file = sub {
        $end_file->();
        $cur_file = shift;
        @stored_chunks = ();
        $show_status->() if $cur_file->is_dir;
        if ($gpg_iter) {
            # catch our gpg iterator up.  we want it to be ahead of us,
            # nothing iteresting is behind us.
            $gpg_iter->next while $gpg_iter->behind_by > 1;
        }
        $file_has_shown_status = 0;
    };

    # records are either Brackup::File (for symlinks, directories, etc), or
    # PositionedChunks, in which case the file can asked of the chunk
    while (my $rec = $chunk_iterator->next) {
        if ($rec->isa("Brackup::File")) {
            $start_file->($rec);
            next;
        }
        my $pchunk = $rec;
        if ($pchunk->file != $cur_file) {
            $start_file->($pchunk->file);
        }

        # have we already stored this chunk before?  (iterative backup)
        my $schunk;
        if ($schunk = $target->stored_chunk_from_inventory($pchunk)) {
            $pchunk->forget_chunkref;
            push @stored_chunks, $schunk;
            next;
        }

        # weird case... have we stored this same pchunk digest in the
        # current comp_chunk we're building?  these aren't caught by
        # the above inventory check, because chunks in a composite
        # chunk aren't added to the inventory until after the the composite
        # chunk has fully grown (because it's not until it's fully grown
        # that we know the handle for it, its digest)
        if ($comp_chunk && ($schunk = $comp_chunk->stored_chunk_from_dup_internal_raw($pchunk))) {
            $pchunk->forget_chunkref;
            push @stored_chunks, $schunk;
            next;
        }

        unless ($file_has_shown_status++) {