view release on metacpan or  search on metacpan

t/unit/cmd_sync.t  view on Meta::CPAN

use List::Util qw/first/;
use Scalar::Util qw/looks_like_number/;

use Test::Spec;
use Test::More;
use Test::Deep;

use Data::Dumper;

use App::MtAws::Journal;
require App::MtAws::Command::Sync;



plan tests => 475;

describe "command" => sub {
	my $j;

	before each => sub {
		$j = App::MtAws::Journal->new(journal_file => 'x', 'root_dir' => 'x' );
	};

	describe "modified processing" => sub {

		my @all_detect = qw/treehash mtime mtime-and-treehash mtime-or-treehash always-positive size-only/; # TODO: fetch from ConfigDefinition
		my @detect_with_mtime = grep /mtime/, @all_detect;
		my @detect_without_mtime = grep !/mtime/, @all_detect;

		describe "is_mtime_differs" => sub {
			it "should work when mtime same" => sub {
				App::MtAws::Command::Sync->expects("file_mtime")->returns(sub{ is shift, 'file1'; 123;})->once;
				ok !App::MtAws::Command::Sync::is_mtime_differs({detect => 'mtime-and-treehash'},{mtime => 123}, 'file1');
			};
			it "should work when mtime greater than" => sub {
				App::MtAws::Command::Sync->expects("file_mtime")->returns(sub{ is shift, 'file1'; 456;})->once;
				ok App::MtAws::Command::Sync::is_mtime_differs({detect => 'mtime-and-treehash'},{mtime => 123}, 'file1');
			};
			it "should work when mtime less than" => sub {
				App::MtAws::Command::Sync->expects("file_mtime")->returns(sub{ is shift, 'file1'; 42;})->once;
				ok App::MtAws::Command::Sync::is_mtime_differs({detect => 'mtime-and-treehash'},{mtime => 123}, 'file1');
			};
			it "should work when mtime is undefined in journal" => sub {
				App::MtAws::Command::Sync->expects("file_mtime")->never;
				ok !App::MtAws::Command::Sync::is_mtime_differs({detect => 'mtime-and-treehash'},{mtime => undef}, 'file1');
			};
			it "should work when detect contains mtime" => sub {
				for (@detect_with_mtime) {
					App::MtAws::Command::Sync->expects("file_mtime")->returns(sub{ is shift, 'file1'; 42;})->once;
					ok App::MtAws::Command::Sync::is_mtime_differs({detect => $_},{mtime => 123}, 'file1');
				}
			};
			it "should work when detect does not contain mtime" => sub {
				for (@detect_without_mtime) {
					App::MtAws::Command::Sync->expects("file_mtime")->never;
					ok ! defined App::MtAws::Command::Sync::is_mtime_differs({detect => $_},{mtime => 123}, 'file1');
				}
			};
		};

		describe "should_upload" => sub {

			it "should define unique constants" => sub {
				ok App::MtAws::Command::Sync::SHOULD_CREATE() != App::MtAws::Command::Sync::SHOULD_TREEHASH();
				ok App::MtAws::Command::Sync::SHOULD_CREATE() != App::MtAws::Command::Sync::SHOULD_NOACTION();

				ok App::MtAws::Command::Sync::SHOULD_CREATE();
				ok App::MtAws::Command::Sync::SHOULD_TREEHASH();
				ok !App::MtAws::Command::Sync::SHOULD_NOACTION(); # one should be FALSE

				# numeric eq only
				ok looks_like_number App::MtAws::Command::Sync::SHOULD_CREATE();
				ok looks_like_number App::MtAws::Command::Sync::SHOULD_TREEHASH();
				ok looks_like_number App::MtAws::Command::Sync::SHOULD_NOACTION();
			};

			it "should almost always return create if file size differs" => sub {
				for (grep $_ ne 'always-positive', @all_detect) {
					App::MtAws::Command::Sync->expects("is_mtime_differs")->never;
					App::MtAws::Command::Sync->expects("file_size")->returns(42)->once;
					is  App::MtAws::Command::Sync::should_upload({detect => $_},{mtime => 123, size => 43}, 'file1'),
						App::MtAws::Command::Sync::SHOULD_CREATE();
				}
			};

			sub test_should_upload
			{
				my ($detect, $mtime_differs, $size_differs, $expected) = @_;
				my $opts = {detect => $detect};
				my $file = {mtime => 123, size => 42};
				if (defined $mtime_differs) {
					App::MtAws::Command::Sync->expects("is_mtime_differs")->returns(sub {
						cmp_deeply [$opts, $file, 'file1'], [@_];
						return $mtime_differs;
					})->once
				} else {
					App::MtAws::Command::Sync->expects("is_mtime_differs")->never;
				}
				if (defined $size_differs) {
					App::MtAws::Command::Sync->expects("file_size")->returns(sub {
						cmp_deeply ['file1'], [@_];
						return $size_differs ? 43 : 42;
					})->once
				} else {
					App::MtAws::Command::Sync->expects("file_size")->never;
				}
				cmp_deeply App::MtAws::Command::Sync::should_upload($opts, $file, 'file1'), $expected;
			}

			describe "detect=mtime" => sub {
				it "should return 'create' when mtime differs" => sub {
					test_should_upload('mtime', 1, 0, App::MtAws::Command::Sync::SHOULD_CREATE());
				};
				it "should return FALSE when mtime same" => sub {
					test_should_upload('mtime', 0, 0, App::MtAws::Command::Sync::SHOULD_NOACTION());
				};
			};

			describe "detect=treehash" => sub {
				it "should return 'treehash' mtime is irrelevant" => sub {
					test_should_upload('treehash', undef, 0, App::MtAws::Command::Sync::SHOULD_TREEHASH());
				};
			};

			describe "detect=mtime-and-treehash" => sub {
				it "should return 'treehash' when mtime differs" => sub {
					test_should_upload('mtime-and-treehash', 1, 0, App::MtAws::Command::Sync::SHOULD_TREEHASH());
				};
				it "should return FALSE when mtime same" => sub {
					test_should_upload('mtime-and-treehash', 0, 0, App::MtAws::Command::Sync::SHOULD_NOACTION());
				};
			};

			describe "detect=mtime-or-treehash" => sub {
				it "should return 'create' when mtime differs" => sub {
					test_should_upload('mtime-or-treehash', 1, 0, App::MtAws::Command::Sync::SHOULD_CREATE());
				};
				it "should return 'treehash' when mtime same" => sub {
					test_should_upload('mtime-or-treehash', 0, 0, App::MtAws::Command::Sync::SHOULD_TREEHASH());
				};
			};

			describe "detect=always-positive" => sub {
				it "should return 'create' always" => sub {
					test_should_upload('always-positive', undef, undef, App::MtAws::Command::Sync::SHOULD_CREATE());
				};
			};

			describe "detect=size-only" => sub {
				it "should return 'create' if size differs" => sub {
					test_should_upload('size-only', undef, 1, App::MtAws::Command::Sync::SHOULD_CREATE());
				};
				it "should return 'no action' if size same" => sub {
					test_should_upload('size-only', undef, 0, App::MtAws::Command::Sync::SHOULD_NOACTION());
				};
			};

			describe "detect is unknown" => sub {
				my $file = {mtime => 123, size => 42};
				App::MtAws::Command::Sync->expects("file_size")->returns(42)->once;
				ok ! defined eval { App::MtAws::Command::Sync::should_upload({detect => 'xyz'}, $file, 'file1'); 1; };
				ok $@ =~ /Invalid detect option in should_upload/;
			}
		};

		describe "next_modified" => sub {
			my $options;
			before each => sub {
				$options = { partsize => 2};
			};

			sub expect_should_upload
			{
				my ($options, $j, $file, $toreturn) = @_;
				App::MtAws::Command::Sync->expects("should_upload")->returns(sub {
					my ($opt, $f, $absfilename) = @_;
					cmp_deeply $opt, $options;
					cmp_deeply $f, $file;
					is $absfilename, $j->absfilename($file->{relfilename});
					return $toreturn;
				})->once;
			}

			sub verify_create_job
			{
				my ($options, $j, $file, $rec) = @_;
				ok $rec->isa('App::MtAws::QueueJob::Upload');
				is $rec->{partsize}, $options->{partsize}*1024*1024;
				is $rec->{relfilename}, $file->{relfilename};
				is $rec->{filename}, $j->absfilename($file->{relfilename});
				ok $rec->{delete_after_upload};
				is $rec->{archive_id}, $file->{archive_id};
			}

			sub verify_treehash_job
			{
				my ($options, $j, $file, $rec) = @_;
				ok $rec->isa('App::MtAws::QueueJob::VerifyAndUpload');
				is $rec->{filename}, $j->absfilename($file->{relfilename});
				is $rec->{relfilename}, $file->{relfilename};
				ok $rec->{delete_after_upload};
				is $rec->{archive_id}, $file->{archive_id};
				is $rec->{treehash}, $file->{treehash};
				is $rec->{partsize}, $options->{partsize}*1024*1024;
			}


			it "should work with zero files" => sub {
				$j->{listing}{existing} = [];
				ok !defined App::MtAws::Command::Sync::next_modified($options, $j);
			};

			it "should work when should_upload returns SHOULD_CREATE" => sub {
				my $file = {relfilename => 'file1', archive_id => 'zz1'};
				$j->{listing}{existing} = [$file];
				$j->_add_filename($file);
				expect_should_upload($options, $j, $file, App::MtAws::Command::Sync::SHOULD_CREATE());
				my $rec = App::MtAws::Command::Sync::next_modified($options, $j);
				verify_create_job($options, $j, $file, $rec);

				is scalar @{ $j->{listing}{existing} }, 0;
				ok !defined (App::MtAws::Command::Sync::next_modified($options, $j));
			};

			it "should work with two files" => sub {
				my $file1 = {relfilename => 'file1', archive_id => 'zz1'};
				my $file2 = {relfilename => 'file2', archive_id => 'zz2'};
				$j->{listing}{existing} = [$file1, $file2];
				$j->_add_filename($file1);
				$j->_add_filename($file2);
				expect_should_upload($options, $j, $file1, App::MtAws::Command::Sync::SHOULD_CREATE());
				my $rec = App::MtAws::Command::Sync::next_modified($options, $j);
				verify_create_job($options, $j, $file1, $rec);

				is scalar @{ $j->{listing}{existing} }, 1;

				expect_should_upload($options, $j, $file2, App::MtAws::Command::Sync::SHOULD_CREATE());
				$rec = App::MtAws::Command::Sync::next_modified($options, $j);
				verify_create_job($options, $j, $file2, $rec);
			};

			it "should work with latest version of file" => sub {
				my $file = {relfilename => 'file1', size => 123};
				$j->{listing}{existing} = [$file];
				$j->_add_filename({relfilename => 'file1', archive_id => 'zz1', size => 123, time => 42, mtime => 111, , treehash => 'abc0'});
				$j->_add_filename(my $r = {relfilename => 'file1', archive_id => 'zz2', size => 123, time => 42, mtime => 113, treehash => 'abc'});
				$j->_add_filename({relfilename => 'file1', archive_id => 'zz3', size => 123, time => 42, mtime => 112, , treehash => 'abc2'});
				expect_should_upload($options, $j, $r, App::MtAws::Command::Sync::SHOULD_TREEHASH());
				my $rec = App::MtAws::Command::Sync::next_modified($options, $j);
				verify_treehash_job($options, $j, $r, $rec);
				is scalar @{ $j->{listing}{existing} }, 0;
			};

			it "should call latest() to get latest version of file" => sub {
				my $file = {relfilename => 'file1', size => 123};
				$j->{listing}{existing} = [$file];
				$j->_add_filename({relfilename => 'file1', archive_id => 'zz1', size => 123, time => 42, mtime => 111, , treehash => 'abc0'});
				$j->_add_filename(my $r = {relfilename => 'file1', archive_id => 'zz2', size => 123, time => 42, mtime => 113, treehash => 'abc'});
				$j->_add_filename({relfilename => 'file1', archive_id => 'zz3', size => 123, time => 42, mtime => 112, , treehash => 'abc2'});
				expect_should_upload($options, $j, $r, App::MtAws::Command::Sync::SHOULD_TREEHASH());
				App::MtAws::Journal->expects("latest")->returns(sub{ is $_[1], "file1"; $r})->once;
				App::MtAws::Command::Sync::next_modified($options, $j);
			};

			it "should work when should_upload returns SHOULD_TREEHASH" => sub {
				my $file = {relfilename => 'file1', archive_id => 'zz1', treehash => 'abcdef'};
				$j->{listing}{existing} = [$file];
				$j->_add_filename($file);
				expect_should_upload($options, $j, $file, App::MtAws::Command::Sync::SHOULD_TREEHASH());
				my $rec = App::MtAws::Command::Sync::next_modified($options, $j);
				verify_treehash_job($options, $j, $file, $rec);

				is scalar @{ $j->{listing}{existing} }, 0;
				ok !defined (App::MtAws::Command::Sync::next_modified($options, $j));
			};

			it "should skip to next file when should_upload returns SHOULD_NOACTION" => sub {
				for (1..10) {
					my $file = {relfilename => "file$_", archive_id => "zz$_"};
					push @{ $j->{listing}{existing} }, $file;
					$j->_add_filename($file);
				}

				my $file;
				App::MtAws::Command::Sync->expects("should_upload")->returns(sub {
					my ($opt, $f, $absfilename) = @_;
					$file = $f;
					return $f->{relfilename} eq 'file7' ? App::MtAws::Command::Sync::SHOULD_CREATE() : App::MtAws::Command::Sync::SHOULD_NOACTION();
				})->exactly(10);

				my $rec = App::MtAws::Command::Sync::next_modified($options, $j);
				verify_create_job($options, $j, $file, $rec);

				is scalar @{ $j->{listing}{existing} }, 3;
				ok !defined App::MtAws::Command::Sync::next_modified($options, $j);
			};

			it "should confess when should_upload returns something else" => sub {
				my $file = {relfilename => 'file1', archive_id => 'zz1'};
				$j->{listing}{existing} = [$file];
				$j->_add_filename($file);
				expect_should_upload($options, $j, $file, 7656348);
				ok !defined eval{ App::MtAws::Command::Sync::next_modified($options, $j); 1};
				ok $@ =~ /Unknown value returned by should_upload/;
			};
		};

	};

	describe "next_new" => sub {
		my $options;
		before each => sub {
			$options = { partsize => 2};
		};
		it "should work with one file" => sub {
			$j->{listing}{new} = [{relfilename => 'file1'}];
			my $rec = App::MtAws::Command::Sync::next_new($options, $j);
			ok $rec->isa('App::MtAws::QueueJob::Upload');
			is $rec->{partsize}, $options->{partsize}*1024*1024;
			is $rec->{relfilename}, 'file1';
			is $rec->{filename}, $j->absfilename('file1');
			ok !$rec->{delete_after_upload};
			is scalar @{ $j->{listing}{new} }, 0;
			ok !defined (App::MtAws::Command::Sync::next_new($options, $j));
		};
		it "should work with two files" => sub {
			$j->{listing}{new} = [{relfilename => 'file1'}, {relfilename => 'file2'}];
			my $rec = App::MtAws::Command::Sync::next_new($options, $j);
			ok $rec->isa('App::MtAws::QueueJob::Upload');
			is $rec->{relfilename}, 'file1';
			is scalar @{ $j->{listing}{new} }, 1;
			$rec = App::MtAws::Command::Sync::next_new($options, $j);
			is $rec->{relfilename}, 'file2';
		};
		it "should work with zero files" => sub {
			$j->{listing}{new} = [];
			ok ! defined( App::MtAws::Command::Sync::next_new($options, $j) );
		};
	};

	describe "next_missing" => sub {
		my $options;
		before each => sub {
			$options = { };
		};
		it "should work with one file" => sub {
			my $r = {relfilename => 'file1', archive_id => 'somearchive'};
			$j->{listing}{missing} = [{relfilename => $r->{relfilename}}];
			$j->_add_filename($r);
			my $rec = App::MtAws::Command::Sync::next_missing($options, $j);
			ok $rec->isa('App::MtAws::QueueJob::Delete');
			is $rec->{archive_id}, 'somearchive';
			is $rec->{relfilename}, 'file1';
			is scalar @{ $j->{listing}{missing} }, 0;
			ok !defined (App::MtAws::Command::Sync::next_missing($options, $j));
		};
		it "should work with two files" => sub {
			for ({relfilename => 'file1', archive_id => 'somearchive1'}, {relfilename => 'file2', archive_id => 'somearchive2'}) {
				push @{ $j->{listing}{missing} }, {relfilename => $_->{relfilename}};
				$j->_add_filename($_);
			}
			my $rec = App::MtAws::Command::Sync::next_missing($options, $j);
			ok $rec->isa('App::MtAws::QueueJob::Delete');
			is $rec->{archive_id}, 'somearchive1';
			is $rec->{relfilename}, 'file1';
			is scalar @{ $j->{listing}{missing} }, 1;
			$rec = App::MtAws::Command::Sync::next_missing($options, $j);
			is $rec->{archive_id}, 'somearchive2';
			is $rec->{relfilename}, 'file2';
		};
		it "should work with zero files" => sub {
			$j->{listing}{missing} = [];
			ok ! defined( App::MtAws::Command::Sync::next_missing($options, $j) );
		};
		it "should work with latest version of file" => sub {
			my $r = {relfilename => 'file1'};
			$j->{listing}{missing} = [$r];
			$j->_add_filename({relfilename => 'file1', archive_id => 'zz1', size => 123, time => 42, mtime => 111});
			$j->_add_filename({relfilename => 'file1', archive_id => 'zz2', size => 123, time => 42, mtime => 113});
			$j->_add_filename({relfilename => 'file1', archive_id => 'zz3', size => 123, time => 42, mtime => 112});
			my $rec = App::MtAws::Command::Sync::next_missing($options, $j);
			ok $rec->isa('App::MtAws::QueueJob::Delete');

t/unit/cmd_sync.t  view on Meta::CPAN

				$cb->();
			});
		}

		sub expect_journal_init
		{
			my ($options, $j, $read_files_mode) = @_;
			$j->expects("read_journal")->returns_ordered(sub{
				shift;
				cmp_deeply({@_}, {should_exist => 0});
			})->once;#returns(sub{ is ++shift->{_stage}, 1 })
			App::MtAws::Journal->expects("read_files")->returns_ordered(sub {
				shift;
				cmp_deeply [@_], [$read_files_mode, $options->{'max-number-of-files'}];
			})->once;
			App::MtAws::Journal->expects("open_for_write")->returns_ordered->once;
		}

		sub expect_fork_engine
		{
			App::MtAws::Command::Sync->expects("fork_engine")->returns_ordered(sub {
				bless { parent_worker =>
					bless {}, 'App::MtAws::ParentWorker'
				}, 'App::MtAws::ForkEngine';
			})->once;
		}

		sub expect_journal_close
		{
			App::MtAws::Journal->expects("close_for_write")->returns_ordered->once;
		}

		sub expect_process_task
		{
			my ($j, $cb) = @_;
			App::MtAws::ParentWorker->expects("process_task")->returns_ordered(sub {
				my ($self, $job, $journal) = @_;
				ok $self->isa('App::MtAws::ParentWorker');
				is $journal, $j;
				$cb->($job);
			} )->once;
		}

		it "should work with new" => sub {
			my $options = { 'max-number-of-files' => 10, partsize => 2, new => 1 };
			ordered_test sub {
				expect_with_forks;
				expect_journal_init($options, $j, {new=>1});
				expect_fork_engine;
				my @files = qw/file1 file2 file3 file4/;

				expect_process_task($j, sub {
					my ($job) = @_;
					ok $job->isa('App::MtAws::QueueJob::Iterator');
					ok ! keys %{$job->{jobs}}; # no pending jobs
					my $itt = $job->{iterator}->()->{iterator}; # resolve iterator of iterators
					for (@files) {
						my $task = $itt->();
						is $task->{relfilename}, $_;
						is $task->{partsize}, $options->{partsize}*1024*1024;
						ok ! $task->{delete_after_upload};
						ok $task->isa('App::MtAws::QueueJob::Upload');
					}
					return (1)
				});

				expect_journal_close;
				$j->{listing}{existing} = [];
				$j->{listing}{new} = [ map { relfilename => $_ }, @files ];

				App::MtAws::Command::Sync::run($options, $j);
			};
		};

		it "should work with replace-modified" => sub {
			my $options = { 'max-number-of-files' => 10, partsize => 2, 'replace-modified' => 1, detect => 'mtime-and-treehash' };
			ordered_test sub {
				expect_with_forks;
				expect_journal_init($options, $j, {existing=>1});
				expect_fork_engine;
				my %files = (
					file1 => {size => 123, archive_id => 'a1'},
					file2 => {size => 456, archive_id => 'a2'},
					file3 => {size => 789, archive_id => 'a3'},
					file4 => {size => 42, archive_id => 'a4'}
				);

				expect_process_task($j, sub {
					my ($job) = @_;
					ok $job->isa('App::MtAws::QueueJob::Iterator');
					ok ! keys %{$job->{jobs}}; # no pending jobs
					my $itt = $job->{iterator}->()->{iterator}; # resolve iterator of iterators
					for (sort keys %files) {
						my $task = $itt->();
						is $task->{relfilename}, $_;
						is $task->{partsize}, $options->{partsize}*1024*1024;
						ok $task->{delete_after_upload};
						is $task->{archive_id}, $files{$_}{archive_id};
						ok $task->isa('App::MtAws::QueueJob::Upload');

					}
					return (1)
				});

				expect_journal_close;
				$j->{listing}{new} = [];
				for (sort keys %files) {
					my $r = {relfilename => $_};
					$j->_add_filename({%$r, %{$files{$_}}});
					push @{ $j->{listing}{existing} }, $r;
				}
				App::MtAws::Command::Sync->expects("file_size")->returns(sub {
					my ($file) = @_;
					$file =~ m!([^/]+)$! or confess;
					$files{$1}{size}+1 or confess; # return different filesize
				})->exactly(scalar keys %files);

				App::MtAws::Command::Sync::run($options, $j);
			};
		};

		it "should work with delete-removed" => sub {
			my $options = { 'max-number-of-files' => 10, partsize => 2, 'delete-removed' => 1 };
			ordered_test sub {
				expect_with_forks;
				expect_journal_init($options, $j, {missing=>1});
				expect_fork_engine;
				my %files = (
					file1 => {archive_id => 'z123'},
					file2 => {archive_id => 'z456'},
					file3 => {archive_id => 'z789'},
					file4 => {archive_id => 'z42'}
				);

				expect_process_task($j, sub {
					my ($job) = @_;
					ok $job->isa('App::MtAws::QueueJob::Iterator');
					ok ! keys %{$job->{jobs}}; # no pending jobs
					my $itt = $job->{iterator}->()->{iterator}; # resolve iterator of iterators
					for (sort keys %files) {
						my $task = $itt->();
						is $task->{relfilename}, $_;
						is $task->{archive_id}, $files{$_}{archive_id};
						ok $task->isa('App::MtAws::QueueJob::Delete');
					}
					return (1)
				});

				expect_journal_close;
				$j->{listing}{missing} = [];
				for (sort keys %files) {
					my $r = {relfilename => $_, archive_id => $files{$_}{archive_id}};
					$j->_add_filename($r);
					push @{ $j->{listing}{missing} }, {relfilename => $r->{relfilename}};
				}

				App::MtAws::Command::Sync::run($options, $j);