view release on metacpan or  search on metacpan

lib/EAI/Wrap.pm  view on Meta::CPAN

	EAI::Common::setErrSubject("local archiving");
	for my $histFolder ("historyFolder", "historyFolderUpload") {
		my @filenames = ();
		if ($histFolder eq "historyFolderUpload") {
			@filenames = @{$execute{filesToMoveinHistoryUpload}} if $execute{filesToMoveinHistoryUpload} and @{$execute{filesToMoveinHistoryUpload}};
			$redoDir = ""; # no redoDir for uploads !
		} else {
			@filenames = @{$execute{filesToMoveinHistory}} if $execute{filesToMoveinHistory} and @{$execute{filesToMoveinHistory}};
		}
		for (@filenames) {
			my ($strippedName, $ext) = /(.+)\.(.+?)$/;
			# if done from a redoDir, then add this folder to file (e.g. if done from redo/user specific folder then Filename_20190219_124409.txt becomes Filename_20190219_124409_redo_userspecificfolder_.txt)
			my $cutOffSpec = $archiveTimestamp;
			if ($redoDir) {
				my $redoSpec = $redoDir;
				$redoSpec =~ s/[\/\\\*\|\?:<>"]/_/g;
				$cutOffSpec = $archiveTimestamp.'_'.$redoSpec;
			}
			my $histTarget = $execute{$histFolder}."/".$strippedName."_".$cutOffSpec.".".$ext;
			$logger->info("moving local ".($common{task}{redoFile} ? "re-done " : "")."file $redoDir$_ into $histTarget");
			rename $redoDir.$_, $histTarget or $logger->error("error when moving local file $redoDir$_ into $histTarget: $!".longmess());
		}
	}
	@{$execute{filesToMoveinHistoryUpload}}=();
	@{$execute{filesToMoveinHistory}}=();
}

# removing files
sub deleteFiles () {
	my $logger = get_logger();
	my $redoDir = ($common{task}{redoFile} ? $execute{redoDir}."/" : "");
	EAI::Common::setErrSubject("local cleanup");
	my %alreadyDeleted; # to avoid unnecessary errors when unlinking below
	for my $whatToDelete ("filesToDelete","filesUploadedToDelete") {
		#$redoDir = "" if $whatToDelete eq "filesUploadedToDelete";
		my @filenames = @{$execute{$whatToDelete}} if $execute{$whatToDelete} and @{$execute{$whatToDelete}};
		for (@filenames) {
			unless ($alreadyDeleted{"$redoDir$_"}) {
				$logger->info("removing local ".($common{task}{redoFile} ? "re-done " : "")."file $redoDir$_ ");
				unlink $redoDir.$_ or $logger->error("error when removing local file $redoDir$_ : $!".longmess());
				$alreadyDeleted{"$redoDir$_"} = 1;
			}
		}
		@{$execute{$whatToDelete}}=();
	}
}
1;
__END__

=head1 NAME

EAI::Wrap - framework for easy creation of Enterprise Application Integration tasks

=head1 SYNOPSIS


    # site.config
    %config = (
    	sensitive => {
    			dbSys => {user => "DBuser", pwd => "DBPwd"},
    			ftpSystem1 => {user => "FTPuser", pwd => "FTPPwd", privKey => 'path_to_private_key', hostkey =>'hostkey'},
    		},
    	checkLookup => {"task_script.pl" => {errmailaddress => "test\@test.com", errmailsubject => "testjob failed", timeToCheck => "0800", freqToCheck => "B", logFileToCheck => "test.log", logcheck => "started.*"}},
    	executeOnInit => sub {$execute{addToScriptName} = "doWhateverHereToModifySettings";},
    	folderEnvironmentMapping => {Test => "Test", Dev => "Dev", "" => "Prod"},
    	errmailaddress => 'your@mail.address',
    	errmailsubject => "No errMailSubject defined",
    	fromaddress => 'service@mail.address',
    	smtpServer => "a.mail.server",
    	smtpTimeout => 60,
    	testerrmailaddress => 'your@mail.address',
    	logRootPath => {"" => "C:/dev/EAI/Logs",},
    	historyFolder => {"" => "History",},
    	historyFolderUpload => "HistoryUpload",
    	redoDir => {"" => "redo",},
    	task => {
    		redoTimestampPatternPart => '[\d_]',
    		retrySecondsErr => 60*5,
    		retrySecondsErrAfterXfails => 60*10,
    		retrySecondsXfails => 2,
    		retrySecondsPlanned => 60*15,
    	},
    	DB => {
    		server => {Prod => "ProdServer", Test => "TestServer"},
    		cutoffYr2000 => 60,
    		DSN => 'driver={SQL Server};Server=$DB->{server}{$execute{env}};database=$DB->{database};TrustedConnection=Yes;',
    		schemaName => "dbo",
    	},
    	FTP => {
    		lookups => {
    			ftpSystem1 => {remoteHost => {Test => "TestHost", Prod => "ProdHost"}, port => 5022},
    		},
    		maxConnectionTries => 5,
    		sshInstallationPath => "C:/dev/EAI/putty/PLINK.EXE",
    	},
    	File => {
    		format_defaultsep => "\t",
    		format_thousandsep => ",",
    		format_decimalsep => ".",
    	}
    );

    # task_script.pl
    use EAI::Wrap;
    %common = (
    	FTP => {
    		remoteHost => {"Prod" => "ftp.com", "Test" => "ftp-test.com"},
    		remoteDir => "/reports",
    		port => 22,
    		user => "myuser",
    		privKey => 'C:/keystore/my_private_key.ppk',
    		FTPdebugLevel => 0, # ~(1|2|4|8|16|1024|2048)
    	},
    	DB => {
    		tablename => "ValueTable",
    		deleteBeforeInsertSelector => "rptDate = ?",
    		dontWarnOnNotExistingFields => 1,
    		database => "DWH",
    	},
    	task => {
    		plannedUntil => "2359",
    	},
    );
    @loads = (
    	{
    		File => {
    			filename => "Datafile1.XML",
    			format_XML => 1,
    			format_sep => ',',
    			format_xpathRecordLevel => '//reportGrp/CM1/*',
    			format_fieldXpath => {rptDate => '//rptHdr/rptDat', NotionalVal => 'NotionalVal', tradeRef => 'tradeRefId', UTI => 'UTI'}, 
    			format_header => "rptDate,NotionalVal,tradeRef,UTI",
    		},
    	},
    	{
    		File => {
    			filename => "Datafile2.txt",
    			format_sep => "\t",
    			format_skip => 1,
    			format_header => "rptDate	NotionalVal	tradeRef	UTI",
    		},
    	}
    );
    setupEAIWrap();
    standardLoop();

=head1 DESCRIPTION

EAI::Wrap provides a framework for defining EAI jobs directly in Perl, sparing the creator of low-level tasks as FTP-Fetching, file-parsing and storing into a database.
It also can be used to handle other workflows, like creating files from the database and uploading to FTP-Servers or using other externally provided tools.

The definition is done by first setting up datastructures for configurations and then providing a high-level scripting of the job itself using the provided subs (although any perl code is welcome here!).

EAI::Wrap has a lot of infrastructure already included, like logging using Log4perl, database handling with L<DBI> and L<DBD::ODBC>, FTP services using L<Net::SFTP::Foreign>, file parsing using L<Text::CSV> (text files), L<Data::XLSX::Parser> and L<S...

Furthermore it provides very flexible commandline options, allowing almost all configurations to be set on the commandline.
Commandline options (e.g. additional information passed on with the interactive option) of the task script are fetched at INIT allowing use of options within the configuration, e.g. $opt{process}{interactive_startdate} for a passed start date.

Also the logging configured in C<$ENV{EAI_WRAP_CONFIG_PATH}/log.config> (logfile root path set in C<$ENV{EAI_WRAP_CONFIG_PATH}/site.config>) starts immediately at INIT of the task script, to use a logger, simply make a call to get_logger(). For the l...

There are two accompanying scripts:

L<setDebugLevel.pl> to easily modify the configured log-levels of the task-script itself and all EAI-Wrap modules.

L<checkLogExist.pl> to run checks on the produced logs (at given times using a cron-job or other scheduler) for their existence and certain (starting/finishing) entries, giving error notifications if the check failed.

=head2 API: datastructures for configurations

=over 4

=item %config