ASNMTAP
view release on metacpan or search on metacpan
applications/archive.pl view on Meta::CPAN
$rvOpendir = opendir(DIR, $reportPath);
if ($rvOpendir) {
while ($reportFilename = readdir(DIR)) {
print "Report Filename: <$reportFilename>\n" if ($debug >= 2);
removeOldReportFiles ($removeReportsEpoch, $removeGzipEpoch, $reportPath, $reportFilename);
}
closedir(DIR);
}
}
foreach $dtest (@stest) {
my ($catalogID_uKey, $test) = split(/\#/, $dtest);
($command, undef) = split(/\.pl/, $test);
my ($catalogID, $uKey) = split(/_/, $catalogID_uKey);
unless ( defined $uKey ) {
$uKey = $catalogID;
$catalogID = $CATALOGID;
$catalogID_uKey = $catalogID .'_'. $uKey unless ( $catalogID eq 'CID' );
}
my ( $tWeek, $tYear ) = get_week('yesterday');
$weekFilename = get_year('yesterday') ."w$tWeek-$command-$catalogID_uKey-csv-week.txt";
if (-e "$path/$weekFilename") { unlink ($path.'/'.$weekFilename); }
print "Test : <$dtest>\n" if ($debug);
foreach $filename (@files) {
print "Filename : <$filename>\n" if ($debug >= 2);
catAllCsvFilesYesterdayWeek ($firstDayOfWeekEpoch, $yesterdayEpoch, $catalogID_uKey, $command, $path, $weekFilename, $filename);
removeAllNokgzipCsvSqlErrorWeekFilesOlderThenAndMoveToBackupShare ($gzipEpoch, $removeAllNokEpoch, $removeGzipEpoch, $removeDebugEpoch, $removeWeeksEpoch, $catalogID_uKey, $command, $path, $filename);
}
}
}
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub removeAllNokgzipCsvSqlErrorWeekFilesOlderThenAndMoveToBackupShare {
my ($gzipEpoch, $removeAllNokEpoch, $removeGzipEpoch, $removeDebugEpoch, $removeWeeksEpoch, $catalogID_uKey, $command, $path, $filename) = @_;
my ($datum, $staart) = split(/\-/, $filename, 2);
if ( $staart ) {
if ( $staart eq "all.txt" ) {
if ($datum le get_yearMonthDay($removeAllNokEpoch)) {
if ($debug) {
print "A- <$datum><", get_yearMonthDay($removeAllNokEpoch), "><$path><$filename>\n";
} else {
print EMAILREPORT "A- <$datum><", get_yearMonthDay($removeAllNokEpoch), "> unlink <$path><$filename>\n";
unlink ($path.'/'.$filename);
}
}
} elsif ( $staart eq "$command-$catalogID_uKey-csv.txt" ) {
if ($datum le get_yearMonthDay($gzipEpoch)) {
if ($debug) {
print "C+ <$datum><", get_yearMonthDay($gzipEpoch), "><$path><$filename>\n";
} else {
print EMAILREPORT "C+ <$datum><", get_yearMonthDay($gzipEpoch), "> gzip <$path><$filename>\n";
my ($status, $stdout, $stderr) = call_system ('gzip --force '.$path.'/'.$filename, $debug);
print EMAILREPORT "C+ E R R O R: <$stderr>\n" unless ( $status );
}
}
} elsif ( $staart eq "$command-$catalogID_uKey-csv.txt.gz" ) {
if ($datum le get_yearMonthDay($removeGzipEpoch)) {
if ($debug) {
print "C- <$datum><", get_yearMonthDay($removeGzipEpoch), "><$path><$filename>\n";
} else {
print EMAILREPORT "C- <$datum><", get_yearMonthDay($removeGzipEpoch), "> unlink <$path><$filename>\n";
unlink ($path.'/'.$filename);
}
}
} elsif ( $staart eq "$command-$catalogID_uKey-csv-week.txt" ) {
my ($jaar, $week) = split(/w/, $datum);
my $jaarWeekFilename = int($jaar.$week);
my ( $tWeek, $tYear ) = get_week ('yesterday');
my $jaarWeekYesterday = int(get_year('yesterday'). $tWeek);
if ( $jaarWeekFilename lt $jaarWeekYesterday ) {
if ($debug) {
print "CW+<$jaarWeekYesterday><$jaarWeekFilename><$path><$filename>\n";
} else {
print EMAILREPORT "CW+<$jaarWeekYesterday><$jaarWeekFilename> gzip <$path><$filename>\n";
my ($status, $stdout, $stderr) = call_system ('gzip --force '.$path.'/'.$filename, $debug);
print EMAILREPORT "CW+ E R R O R: <$stderr>\n" unless ( $status );
}
}
} elsif ( $staart eq "$command-$catalogID_uKey-csv-week.txt.gz" ) {
my ($jaar, $week) = split(/w/, $datum);
my $jaarWeekFilename = int($jaar.$week);
my ( $tWeek, $tYear ) = get_week ('-'. $removeGzipWeeksAgo. ' weeks');
my $jaarWeekRemove = int(get_year('-'. $removeGzipWeeksAgo. ' weeks') .$tWeek);
if ( $jaarWeekFilename le $jaarWeekRemove ) {
if ($debug) {
print "CW-<$jaarWeekRemove><$jaarWeekFilename><", get_yearMonthDay($removeWeeksEpoch), "<$path><$filename>\n";
} else {
print EMAILREPORT "CW-<$jaarWeekRemove><$jaarWeekFilename><", get_yearMonthDay($removeWeeksEpoch), " unlink <$path><$filename>\n";
unlink ($path.'/'.$filename);
}
}
} elsif ( $staart eq "$command-$catalogID_uKey.sql" ) {
if ($debug) {
print "S+ <$datum><", get_yearMonthDay($gzipEpoch), "><$path><$filename>\n" if ($datum le get_yearMonthDay($gzipEpoch));
} elsif (! $doDatabase) {
# APE # TODO - REMOVE
# Init parameters
# my ($rv, $dbh, $sql);
# open connection to database and query data
# $rv = 1;
# $dbh = DBI->connect("dbi:mysql:$DATABASE:$SERVERNAMEREADWRITE:$SERVERPORTREADWRITE", "$SERVERUSERREADWRITE", "$SERVERPASSREADWRITE" ) or $rv = errorTrapDBI("Cannot connect to the database", $debug);
# if ($dbh and $rv) {
# $sql = "LOAD DATA LOW_PRIORITY LOCAL INFILE '$path/$filename' INTO TABLE $SERVERTABLEVENTS FIELDS TERMINATED BY ',' ENCLOSED BY '\"' LINES TERMINATED BY '\\n'";
# $dbh->do( $sql ) or $rv = errorTrapDBI("Cannot dbh->do: $sql", $debug);
# if ( $rv ) {
# my $mysqlInfo = $dbh->{mysql_info};
# my ($records, $deleted, $skipped, $warnings) = ($mysqlInfo =~ /^Records:\s+(\d+)\s+Deleted:\s+(\d+)\s+Skipped:\s+(\d+)\s+Warnings: (\d+)$/);
# if ($deleted eq '0' and $skipped eq '0' and $warnings eq '0') {
# print EMAILREPORT "S+ LOAD DATA ... : $records record(s) added for $filename\n";
# my ($status, $stdout, $stderr) = call_system ('gzip --force '.$path.'/'.$filename, $debug);
# print EMAILREPORT "S+ E R R O R: <$stderr>\n" unless ( $status );
# } else {
# print EMAILREPORT "S+ LOAD DATA ... WARNING for $filename: $mysqlInfo, <$records> <$deleted> <$skipped> <$warnings>\n";
# rename("$path/$filename", "$path/$filename-LOAD-DATA-FAILED");
# }
# }
# $dbh->disconnect or $rv = errorTrapDBI("Sorry, the database was unable to add your entry.", $debug);
# }
my $_debug = ( ( $debug eq 'T' ) ? 1 : 0);
my $dbh = CSV_prepare_table ("$path/", $filename, '', $SERVERTABLEVENTS, \@EVENTS, \%EVENTS, \$logger, $_debug);
my $rv = CSV_import_from_table (1, $dbh, $SERVERTABLEVENTS, \@EVENTS, 'id', $doForce, \$logger, $_debug);
if ( $rv ) {
if ($debug) {
print "S+ IMPORT CSV DATA ... OK: ALL records imported from $path/$filename\n";
} else {
print EMAILREPORT "S+ IMPORT CSV DATA ... OK: ALL records imported from $path/$filename\n";
my ($status, $stdout, $stderr) = call_system ('gzip --force '.$path.'/'.$filename, $debug);
print EMAILREPORT "S+ E R R O R: <$stderr>\n" unless ( $status );
}
} else {
if ($debug) {
print "S- IMPORT CSV DATA ... CRITICAL: ZERO records imported from $path/$filename\n";
} else {
print EMAILREPORT "S- IMPORT CSV DATA ... CRITICAL: ZERO records imported from $path/$filename\n";
rename("$path/$filename", "$path/$filename-LOAD-DATA-FAILED");
}
}
CSV_cleanup_table ($dbh, \$logger, $_debug);
}
} elsif ( $staart eq "$command-$catalogID_uKey.sql.gz" ) {
if ($datum le get_yearMonthDay($removeGzipEpoch)) {
if ($debug) {
print "S- <$datum><", get_yearMonthDay($removeGzipEpoch), "><$path><$filename>\n";
} else {
print EMAILREPORT "S- <$datum><", get_yearMonthDay($removeGzipEpoch), "> unlink <$path><$filename>\n";
unlink ($path.'/'.$filename);
}
}
} elsif ( $staart eq "$command-$catalogID_uKey-sql-error.txt" ) {
if ($datum le get_yearMonthDay($removeGzipEpoch)) {
if ($debug) {
print "SE-<$datum><", get_yearMonthDay($removeDebugEpoch), "><$path><$filename>\n";
} else {
print EMAILREPORT "SE-<$datum><", get_yearMonthDay($removeDebugEpoch), "> unlink <$path><$filename>\n";
unlink ($path.'/'.$filename);
}
}
} elsif ( $staart eq "nok.txt" ) {
if ($datum le get_yearMonthDay($removeAllNokEpoch)) {
if ($debug) {
print "N- <$datum><", get_yearMonthDay($removeAllNokEpoch), "><$path><$filename>\n";
} else {
print EMAILREPORT "N- <$datum><", get_yearMonthDay($removeAllNokEpoch), "> unlink <$path><$filename>\n";
unlink ($path.'/'.$filename);
}
}
}
}
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub catAllCsvFilesYesterdayWeek {
my ($firstDayOfWeekEpoch, $yesterdayEpoch, $catalogID_uKey, $command, $path, $weekFilename, $filename) = @_;
for (my $loop = $firstDayOfWeekEpoch; $loop <= $yesterdayEpoch; $loop += 86400) {
if ($filename eq get_yearMonthDay($loop)."-$command-$catalogID_uKey-csv.txt") {
my $rvOpen = open(CAT, ">>$path/$weekFilename");
if ($rvOpen) {
$rvOpen = open(CSV, "$path/$filename");
if ($rvOpen) {
while (<CSV>) {
chomp;
unless ( /^#/ ) {
my $dummy = $_;
$dummy =~ s/\ {1,}//g;
if ($dummy ne '') { print CAT $_, "\n"; }
}
}
close(CSV);
my ( $tWeek, $tYear ) = get_week ('yesterday');
print "WF <week$tWeek><$filename>\nW <$path/$weekFilename>\n" if ($debug);
} else {
print "Cannot open $filename!\n";
}
close(CAT);
} else {
print "Cannot open $filename!\n";
}
}
}
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub gzipOrRemoveHttpDumpDebug {
my ($gzipDebugEpoch, $removeDebugEpoch, $debugPath, $debugFilename) = @_;
my ($suffix, $extentie, $datum, $restant);
print "<$debugFilename>\n" if ($debug);
my $_debugFilename = reverse $debugFilename;
my ($_suffix, $_extentie) = reverse split(/\./, $_debugFilename, 2);
$suffix = reverse $_suffix;
$extentie = reverse $_extentie;
($datum, $restant) = split(/\-/, $suffix, 2);
if (defined $restant) {
$datum = substr($datum, 0, 8);
if ( $extentie ) {
if ( $extentie eq 'htm' ) {
if ($datum le get_yearMonthDay($gzipDebugEpoch)) {
if ($debug) {
print "HT+<$datum><".get_yearMonthDay($gzipDebugEpoch)."><$debugPath><$debugFilename>\n";
} else {
print EMAILREPORT "HT+<$datum><".get_yearMonthDay($gzipDebugEpoch)."> gzip <$debugPath><$debugFilename>\n";
my ($status, $stdout, $stderr) = call_system ('gzip --force '.$debugPath.'/'.$debugFilename, $debug);
print EMAILREPORT "HT+ E R R O R: <$stderr>\n" unless ( $status );
}
}
} elsif ( $extentie eq 'htm.gz' ) {
if ($datum le get_yearMonthDay($removeDebugEpoch)) {
if ($debug) {
print "HT-<$datum><".get_yearMonthDay($removeDebugEpoch)."><$debugPath><$debugFilename>\n";
} else {
print EMAILREPORT "HT-<$datum><".get_yearMonthDay($removeDebugEpoch)."> unlink <$debugPath><$debugFilename>\n";
unlink ($debugPath.'/'.$debugFilename);
}
}
}
}
}
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub removeCgisessFiles {
my ($removeCgisessEpoch) = @_;
my $emailreport = "\nRemove cgisess files:\n---------------------\n";
if ( $debug ) { print "$emailreport"; } else { print EMAILREPORT "$emailreport"; }
my @cgisessPathFilenames = glob("$CGISESSPATH/cgisess_*");
foreach my $cgisessPathFilename (@cgisessPathFilenames) {
my (undef, $cgisessFilename) = split (/^$CGISESSPATH\//, $cgisessPathFilename);
my (undef, $sessionID) = split (/^cgisess_/, $cgisessFilename);
print "Filename : <$cgisessFilename><$sessionID>\n" if ($debug >= 2);
my ($sessionExists, %session) = get_session_param ($sessionID, $CGISESSPATH, $cgisessFilename, $debug);
if ( $sessionExists ) {
if (defined $session{ASNMTAP}) {
if ($session{ASNMTAP} eq 'LEXY') {
print " : <$removeCgisessEpoch><" .$session{_SESSION_CTIME}. ">\n" if ($debug >= 2);
if ($removeCgisessEpoch > $session{_SESSION_CTIME}) {
if ($debug) {
print "CS <$cgisessPathFilename><" .$session{_SESSION_CTIME}. ">\n";
} else {
print EMAILREPORT "CS unlink <$cgisessPathFilename><" .$session{_SESSION_CTIME}. ">\n";
my ($status, $stdout, $stderr) = call_system ('rm -f '.$cgisessPathFilename, $debug); # unlink ($cgisessPathFilename);
}
} else {
print "CS-<$cgisessPathFilename><$removeCgisessEpoch><" .$session{_SESSION_CTIME}. ">\n" if ($debug >= 2);
}
} else {
print "CS-<$cgisessPathFilename> ASNMTAP not LEXY>\n" if ($debug >= 2);
}
} else {
if ($removeCgisessEpoch > $session{_SESSION_CTIME}) {
if ($debug) {
print "CS <$cgisessPathFilename><" .$session{_SESSION_CTIME}. ">\n";
} else {
print EMAILREPORT "CS unlink <$cgisessPathFilename><" .$session{_SESSION_CTIME}. ">\n";
unlink ($cgisessPathFilename);
}
} else {
print "CS-<$cgisessPathFilename> ASNMTAP not LEXY>\n" if ($debug >= 2);
}
}
}
}
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub removeOldReportFiles {
my ($removeReportsEpoch, $removeGzipEpoch, $reportPath, $reportFilename) = @_;
my ($suffix, $prefix, $datum, $plugin, $restant, $extentie);
($suffix, $prefix) = split(/\.pl/, $reportFilename, 2);
($datum, $plugin) = split(/\-/, $suffix, 2) if (defined $suffix);
($restant, $extentie) = split(/\./, $prefix, 2) if (defined $prefix);
if ($debug) {
print "<$reportFilename>";
if ($debug >= 2) {
print " S <$suffix>, P <$prefix>" if (defined $prefix);
print " D <$datum>, P <$plugin>" if (defined $plugin);
print " R <$restant>, E <$extentie>" if (defined $extentie);
}
print "\n";
}
if (defined $restant) {
$datum = substr($datum, 0, 8);
if ($extentie eq 'pdf') {
if ($datum le get_yearMonthDay($removeReportsEpoch)) {
if ($debug) {
print "RP-<$datum><".get_yearMonthDay($removeReportsEpoch)."><$reportPath><$reportFilename>\n";
} else {
print EMAILREPORT "RP-<$datum><".get_yearMonthDay($removeReportsEpoch)."> unlink <$reportPath><$reportFilename>\n";
unlink ($reportPath.'/'.$reportFilename);
}
} elsif ($restant =~ /\-Day_\w+\-id_\d+$/) {
if ($datum le get_yearMonthDay($removeGzipEpoch)) {
if ($debug) {
print "RP-<$datum><".get_yearMonthDay($removeReportsEpoch)."><$reportPath><$reportFilename>\n";
( run in 1.060 second using v1.01-cache-2.11-cpan-5a3173703d6 )