ASNMTAP
view release on metacpan or search on metacpan
applications/archive.pl view on Meta::CPAN
#!/usr/bin/env perl
# ---------------------------------------------------------------------------------------------------------
# © Copyright 2003-2011 Alex Peeters [alex.peeters@citap.be]
# ---------------------------------------------------------------------------------------------------------
# 2011/mm/dd, v3.002.003, archive.pl for ASNMTAP::Applications
# ---------------------------------------------------------------------------------------------------------
use strict;
use warnings; # Must be used in test mode only. This reduces a little process speed
#use diagnostics; # Must be used in test mode only. This reduces a lot of process speed
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
BEGIN { if ( $ENV{ASNMTAP_PERL5LIB} ) { eval 'use lib ( "$ENV{ASNMTAP_PERL5LIB}" )'; } }
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
use DBI;
use Time::Local;
use Getopt::Long;
use Date::Calc qw(Date_to_Time Monday_of_Week);
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
use ASNMTAP::Time v3.002.003;
use ASNMTAP::Time qw(&get_epoch &get_wday &get_yearMonthDay &get_year &get_month &get_day &get_week);
use ASNMTAP::Asnmtap::Applications v3.002.003;
use ASNMTAP::Asnmtap::Applications qw(:APPLICATIONS :ARCHIVE :DBARCHIVE $SERVERTABLPLUGINS $SERVERTABLVIEWS $SERVERTABLDISPLAYDMNS $SERVERTABLCRONTABS $SERVERTABLCLLCTRDMNS $SERVERTABLSERVERS );
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
use vars qw($opt_A $opt_c $opt_r $opt_d $opt_y $opt_f $opt_D $opt_V $opt_h $PROGNAME);
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
$PROGNAME = "archive.pl";
my $prgtext = "Archiver for the '$APPLICATION'";
my $version = do { my @r = (q$Revision: 3.002.003$ =~ /\d+/g); sprintf "%d."."%03d" x $#r, @r }; # must be all on one line or MakeMaker will get confused.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
my $doCgisess = 1; # default
my $doReports = 1; # default
my $doDatabase = 0; # default
my $doYearsAgo = -1; # default
my $doForce = 0; # default
my $debug = 0; # default
#------------------------------------------------------------------------
# Don't edit below here unless you know what you are doing. -------------
#------------------------------------------------------------------------
my $archivelist;
my $gzipDaysAgo = 8; # GZIP files older then n date
my $gzipDebugDaysAgo = 3; # GZIP files older then n days ago
my $removeGzipDaysAgo = 31; # Remove files older then n days ago
my $removeAllNokDaysAgo = 8; # Remove files older then n days ago
my $removeDebugDaysAgo = 31; # Remove files older then n days ago
my $removeGzipWeeksAgo = 53; # Remove files older then n weeks ago
my $removeCgisessDaysAgo = 2; # Remove files older then n days ago
my $removeReportWeeksAgo = 53; # Remove files older then n weeks ago
my $gzipEpoch = get_epoch ('-'. $gzipDaysAgo .' days'); # GZIP files older then n date
my $gzipDebugEpoch = get_epoch ('-'. $gzipDebugDaysAgo .' days'); # GZIP files older then n date
my $removeAllNokEpoch = get_epoch ('-'. $removeAllNokDaysAgo .' days'); # Remove files older then n days ago
my $removeGzipEpoch = get_epoch ('-'. $removeGzipDaysAgo .' days'); # Remove files older then n days ago
my $removeDebugEpoch = get_epoch ('-'. $removeDebugDaysAgo .' days'); # Remove files older then n days ago
my $removeWeeksEpoch = get_epoch ('-'. $removeGzipWeeksAgo .' weeks'); # Remove files older then n weeks ago
applications/archive.pl view on Meta::CPAN
$dbh->do( $sql ) or $rv = errorTrapDBI("Cannot dbh->do: $sql", $debug);
}
if ($rv) {
$sql = 'ALTER TABLE `'. $SERVERTABLEVENTS .'_'. $year .'` ENGINE=MERGE UNION=(`'. $SERVERTABLEVENTS .'_'. $year .'_01`, `'. $SERVERTABLEVENTS .'_'. $year .'_02`, `'. $SERVERTABLEVENTS .'_'. $year .'_03`, `'. $SERVERTABLEVENTS .'_'. $year .'...
$dbh->do( $sql ) or $rv = errorTrapDBI("Cannot dbh->do: $sql", $debug);
}
if ($rv) { print EMAILREPORT "MERGED\n\n"; } else { print EMAILREPORT "NOT MERGED, PLEASE VERIFY '$sql'\n\n"; }
}
foreach my $quarter (1..4) {
if ($debug) {
print "\nTable: '$SERVERTABLEVENTS', Year: '$year' Quarter: 'Q$quarter', Status: MERGE\n";
} else {
print EMAILREPORT "\nTable: '$SERVERTABLEVENTS', Year: '$year' Quarter: 'Q$quarter', Status: MERGE\n";
$sql = 'DROP TABLE IF EXISTS `'. $SERVERTABLEVENTS .'_'. $year .'_Q'. $quarter .'`';
$dbh->do( $sql ) or $rv = errorTrapDBI("Cannot dbh->do: $sql", $debug);
if ($rv) {
$sql = 'CREATE TABLE IF NOT EXISTS `'. $SERVERTABLEVENTS .'_'. $year .'_Q'. $quarter .'` LIKE `'. $SERVERTABLEVENTS .'_'. $year .'_'. sprintf ("%02d", ($quarter * 3 ) - 2) .'`';
$dbh->do( $sql ) or $rv = errorTrapDBI("Cannot dbh->do: $sql", $debug);
}
if ($rv) {
$sql = 'ALTER TABLE `'. $SERVERTABLEVENTS .'_'. $year .'_Q'. $quarter .'` ENGINE=MERGE UNION=(`'. $SERVERTABLEVENTS .'_'. $year .'_'. sprintf ("%02d", ($quarter * 3 ) - 2) .'`, `'. $SERVERTABLEVENTS .'_'. $year .'_'. sprintf ("%02d", ($qu...
$dbh->do( $sql ) or $rv = errorTrapDBI("Cannot dbh->do: $sql", $debug);
}
if ($rv) { print EMAILREPORT "MERGED\n\n"; } else { print EMAILREPORT "NOT MERGED, PLEASE VERIFY '$sql'\n\n"; }
}
}
}
$sql = "CREATE TABLE IF NOT EXISTS `". $SERVERTABLCOMMENTS .'_'. $year ."` LIKE `$SERVERTABLCOMMENTS`";
$rv = ! checkTableDBI ($dbh, $DATABASE, $SERVERTABLCOMMENTS .'_'. $year, 'check', 'status', 'OK');
if ($rv) {
if ($debug) {
print "\nTable: '$SERVERTABLCOMMENTS', Year: '$year'\n<$sql>\n";
} else {
print EMAILREPORT "\nTable: '$SERVERTABLCOMMENTS', Year: '$year', Status: ";
$dbh->do( $sql ) or $rv = errorTrapDBI("Cannot dbh->do: $sql", $debug);
$rv = checkTableDBI ($dbh, $DATABASE, $SERVERTABLCOMMENTS .'_'. $year, 'check', 'status', 'OK');
if ($rv) { print EMAILREPORT "Created\n\n"; } else { print EMAILREPORT "NOT CREATED, PLEASE VERIFY\n\n"; }
}
} else {
print "Table: '$SERVERTABLCOMMENTS', Year: '$year', Status: ALREADY CREATED\n\n" if ($debug);
}
$dbh->disconnect or $rv = errorTrapDBI("Sorry, the database was unable to add your entry.", $debug);
}
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub doBackupCsvSqlErrorWeekDebugReport {
my ($RESULTSPATH, $DEBUGDIR, $REPORTDIR, $gzipEpoch, $removeAllNokEpoch, $removeGzipEpoch, $removeDebugEpoch, $removeReportsEpoch, $removeWeeksEpoch, $firstDayOfWeekEpoch, $yesterdayEpoch, $currentEpoch) = @_;
print EMAILREPORT "\nDo backup, csv, sql, error, week, and debug files:\n--------------------------------------------------\n" unless ( $debug );
my ($darchivelist, $dtest, $pagedir, $ttest, $command, $rvOpendir, $path, $filename, $debugPath, $debugFilename, $reportPath, $reportFilename, $weekFilename);
my @files = ();
foreach $darchivelist (@archivelisttable) {
($pagedir, $ttest) = split(/\#/, $darchivelist, 2);
my @stest = split(/\|/, $ttest);
$path = $RESULTSPATH .'/'. $pagedir;
$debugPath = $path .'/'. $DEBUGDIR;
$reportPath = $path .'/'. $REPORTDIR;
if ($debug) {
print "\n", "<$RESULTSPATH><$pagedir><$path><$DEBUGDIR><$REPORTDIR>\n"
} else {
print EMAILREPORT "\nPlugin: '$ttest', results directory: '$path'\n";
}
$rvOpendir = opendir(DIR, $path);
if ($rvOpendir) {
@files = readdir(DIR);
closedir(DIR);
}
if (-e $debugPath) {
$rvOpendir = opendir(DIR, $debugPath);
if ($rvOpendir) {
while ($debugFilename = readdir(DIR)) {
print "Debug Filename: <$debugFilename>\n" if ($debug >= 2);
gzipOrRemoveHttpDumpDebug ($gzipDebugEpoch, $removeDebugEpoch, $debugPath, $debugFilename);
}
closedir(DIR);
}
}
if (-e $reportPath) {
$rvOpendir = opendir(DIR, $reportPath);
if ($rvOpendir) {
while ($reportFilename = readdir(DIR)) {
print "Report Filename: <$reportFilename>\n" if ($debug >= 2);
removeOldReportFiles ($removeReportsEpoch, $removeGzipEpoch, $reportPath, $reportFilename);
}
closedir(DIR);
}
}
foreach $dtest (@stest) {
my ($catalogID_uKey, $test) = split(/\#/, $dtest);
($command, undef) = split(/\.pl/, $test);
my ($catalogID, $uKey) = split(/_/, $catalogID_uKey);
unless ( defined $uKey ) {
$uKey = $catalogID;
$catalogID = $CATALOGID;
$catalogID_uKey = $catalogID .'_'. $uKey unless ( $catalogID eq 'CID' );
}
my ( $tWeek, $tYear ) = get_week('yesterday');
$weekFilename = get_year('yesterday') ."w$tWeek-$command-$catalogID_uKey-csv-week.txt";
if (-e "$path/$weekFilename") { unlink ($path.'/'.$weekFilename); }
print "Test : <$dtest>\n" if ($debug);
foreach $filename (@files) {
print "Filename : <$filename>\n" if ($debug >= 2);
catAllCsvFilesYesterdayWeek ($firstDayOfWeekEpoch, $yesterdayEpoch, $catalogID_uKey, $command, $path, $weekFilename, $filename);
removeAllNokgzipCsvSqlErrorWeekFilesOlderThenAndMoveToBackupShare ($gzipEpoch, $removeAllNokEpoch, $removeGzipEpoch, $removeDebugEpoch, $removeWeeksEpoch, $catalogID_uKey, $command, $path, $filename);
}
}
}
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub removeAllNokgzipCsvSqlErrorWeekFilesOlderThenAndMoveToBackupShare {
my ($gzipEpoch, $removeAllNokEpoch, $removeGzipEpoch, $removeDebugEpoch, $removeWeeksEpoch, $catalogID_uKey, $command, $path, $filename) = @_;
my ($datum, $staart) = split(/\-/, $filename, 2);
if ( $staart ) {
if ( $staart eq "all.txt" ) {
if ($datum le get_yearMonthDay($removeAllNokEpoch)) {
if ($debug) {
print "A- <$datum><", get_yearMonthDay($removeAllNokEpoch), "><$path><$filename>\n";
} else {
print EMAILREPORT "A- <$datum><", get_yearMonthDay($removeAllNokEpoch), "> unlink <$path><$filename>\n";
unlink ($path.'/'.$filename);
}
}
} elsif ( $staart eq "$command-$catalogID_uKey-csv.txt" ) {
if ($datum le get_yearMonthDay($gzipEpoch)) {
if ($debug) {
print "C+ <$datum><", get_yearMonthDay($gzipEpoch), "><$path><$filename>\n";
} else {
print EMAILREPORT "C+ <$datum><", get_yearMonthDay($gzipEpoch), "> gzip <$path><$filename>\n";
my ($status, $stdout, $stderr) = call_system ('gzip --force '.$path.'/'.$filename, $debug);
print EMAILREPORT "C+ E R R O R: <$stderr>\n" unless ( $status );
}
}
} elsif ( $staart eq "$command-$catalogID_uKey-csv.txt.gz" ) {
if ($datum le get_yearMonthDay($removeGzipEpoch)) {
if ($debug) {
print "C- <$datum><", get_yearMonthDay($removeGzipEpoch), "><$path><$filename>\n";
} else {
print EMAILREPORT "C- <$datum><", get_yearMonthDay($removeGzipEpoch), "> unlink <$path><$filename>\n";
unlink ($path.'/'.$filename);
}
}
} elsif ( $staart eq "$command-$catalogID_uKey-csv-week.txt" ) {
my ($jaar, $week) = split(/w/, $datum);
my $jaarWeekFilename = int($jaar.$week);
my ( $tWeek, $tYear ) = get_week ('yesterday');
my $jaarWeekYesterday = int(get_year('yesterday'). $tWeek);
if ( $jaarWeekFilename lt $jaarWeekYesterday ) {
if ($debug) {
print "CW+<$jaarWeekYesterday><$jaarWeekFilename><$path><$filename>\n";
} else {
print EMAILREPORT "CW+<$jaarWeekYesterday><$jaarWeekFilename> gzip <$path><$filename>\n";
my ($status, $stdout, $stderr) = call_system ('gzip --force '.$path.'/'.$filename, $debug);
print EMAILREPORT "CW+ E R R O R: <$stderr>\n" unless ( $status );
}
( run in 2.141 seconds using v1.01-cache-2.11-cpan-39bf76dae61 )