AmberDB
view release on metacpan or search on metacpan
lib/AmberDB/Tools/Maintain.pm view on Meta::CPAN
my ( $self, $tableid ) = @_;
my $adb = $self->{_adb} or return;
$tableid or return;
my $table_path = $adb->table_path($tableid);
my $i = 1;
if ( -e "$table_path.csv" ) {
my $day_id = $adb->day_id;
rename( "$table_path.csv", "$table_path-$day_id.csv" );
}
# cevirme islemlerini yap
my $tie_path = "${table_path}.$adb->{db_ext}";
return 1 unless -e $tie_path;
$adb->table_read($tie_path) or return 1;
my @records;
$adb->recs_scan(
$tie_path,
sub {
my ( $k, $v ) = @_;
my @fields = $adb->db_decode( $v );
push @records, [ $k, $adb->tsv_encode( @fields ) ];
$self->{say} .= "$i. $k ID record converted.\n\n";
$i++;
}
);
@records = sort { $a->[0] <=> $b->[0] } @records;
open my $fh, ">", "$table_path.csv" or do {
cluck "[DB_TOOL] Could not open $table_path.csv: $!\n";
return;
};
foreach my $record ( @records ) {
print $fh "$record->[0]\t$record->[1]\n";
}
close $fh;
$adb->table_close($tie_path);
return 1;
}
# $tools->{ISO2UTF} = 1;
# my $ok = $tools->csv2tie("file");
# ---------------------------------------------------------------------
sub csv2tie {
my ( $self, $tableid ) = @_;
my $adb = $self->{_adb} or return;
$tableid or return;
# file paths
my $table_path = $adb->table_path($tableid);
my $file_path = "${table_path}.$adb->{db_ext}";
my $csv_path = "$table_path.csv";
return unless -e $csv_path;
# backup with timestamp if exists
if ( -e "${table_path}.$adb->{db_ext}" ) {
my $sec_id = $adb->second_id;
rename( "${table_path}.$adb->{db_ext}",
"${table_path}-$sec_id.$adb->{db_ext}" );
unlink("${table_path}.$adb->{db_ext}");
}
# perform conversion operations
my $i = 1;
my @records;
open my $FH, "<", $csv_path or do {
cluck "[DB_TOOL] Could not open $csv_path: $!\n";
return;
};
$adb->table_write($file_path) or do {
close $FH;
cluck "[DB_TOOL] Could not open $file_path for writing.\n";
return;
};
my @chunk;
while ( my $record = <$FH> ) {
chomp($record);
$record =~ s/\r$//;
my (@fields) = $adb->db_decode($record);
push @chunk, [@fields];
if ( @chunk >= 2000 ) {
$adb->recs_put( $file_path, @chunk );
@chunk = ();
}
# collect into list for indexing
push @records, [@fields];
$self->{say} .= "$i. Record ID $fields[0] converted.\n\n" if $i % 1000 == 0;
$i++;
}
if ( @chunk ) {
$adb->recs_put( $file_path, @chunk );
}
close $FH;
$adb->table_close($file_path);
$self->set_index( $tableid, @records );
return 1;
}
# my @tables = $tools->dir_tables();
# ------------------------------------------------
sub dir_tables {
my ( $self, $dir ) = @_;
my $adb = $self->{_adb} or return;
$dir or return;
my $target_dir = File::Spec->catdir( $adb->path('dbase_dir') || ".", $dir );
my $ext = $adb->{db_ext} || "db";
my @names = $adb->dir_files( $target_dir, "*.$ext", full_path => 0 );
my %all_tables = map { /^(.+)\.\Q$ext\E$/i ? ( $1 => 1 ) : () } @names;
lib/AmberDB/Tools/Maintain.pm view on Meta::CPAN
# Reverse operation
rename( $tie_path, $tie_back );
open $FH, "<", $csv_path or do {
cluck "[DB_TOOL] Could not open $csv_path for reading: $!\n";
return;
};
$adb->table_write($tie_path);
my @chunk;
while ( my $record = <$FH> ) {
chomp($record);
$record =~ s/\r$//;
my @fields = $adb->db_decode($record);
push @chunk, [@fields];
if ( @chunk >= 2000 ) {
my $ok = $adb->recs_put( $tie_path, @chunk );
$count->{tie2} += scalar @chunk if $ok;
@chunk = ();
}
}
if ( @chunk ) {
my $ok = $adb->recs_put( $tie_path, @chunk );
$count->{tie2} += scalar @chunk if $ok;
}
close $FH;
$adb->table_close($tie_path);
$adb->table_attr( $tableid, $table_info );
$self->set_index( $tableid, @records ) if $reindex;
$self->{say} .= "Vacuum completed. Record counts: Tie old: $count->{tie1}, CSV: $count->{csv1}, Tie new: $count->{tie2}\n";
return 1;
}
# my $tables_hash = $tools->all_tables(); # scalar context -> grouped hashref
# my @table_list = $tools->all_tables(); # list context -> flat array of table IDs
# ------------------------------------------------
sub all_tables {
my ($self) = @_;
my $adb = $self->{_adb} or return;
my @all_tables;
my %all_tables;
my $dbase_dir = $adb->path('dbase_dir') || ".";
my $year_dir = $adb->path('year_dir') || "";
my $schema_dir = $adb->path('schema_dir') || "";
# 1. Simple Mode: Single flat directory scan for files matching configured db_ext
if ( $adb->config('use_simple') ) {
my $ext = $adb->{db_ext} || "db";
my @files = $adb->dir_files( $dbase_dir, "*.$ext", full_path => 0 );
@all_tables = map { /^([a-z0-9_]+)\.\Q$ext\E$/i ? $1 : () } @files;
}
# 2. Standard Structured Mode: Multi-directory scan (table/ and year directories) for .db files
else {
my $tbl_dir = File::Spec->catdir( $dbase_dir, 'table' );
push @all_tables, $adb->dir_files( $tbl_dir, "*.db", full_path => 0 ) if -d $tbl_dir;
my %seen_dirs = ( "table" => 1, "schema" => 1, "backup" => 1, "lock" => 1, "ramdisk" => 1, "journal" => 1, "session" => 1, "config" => 1 );
if ($year_dir) {
my $yd_path = File::Spec->catdir( $dbase_dir, $year_dir );
push @all_tables, $adb->dir_files( $yd_path, "*.db", full_path => 0 );
$seen_dirs{$year_dir} = 1;
}
# Auto-discover any 4-digit year directories under $dbase_dir (e.g. 2024, 2025, 2026)
if ( -d $dbase_dir ) {
my @year_dirs = $adb->dir_files( $dbase_dir, qr/^\d{4}$/, full_path => 0, files_only => 0 );
foreach my $yd (@year_dirs) {
next if $seen_dirs{$yd} || !-d File::Spec->catdir( $dbase_dir, $yd );
my $yd_path = File::Spec->catdir( $dbase_dir, $yd );
push @all_tables, $adb->dir_files( $yd_path, "*.db", full_path => 0 );
}
}
@all_tables = map { /^([a-z0-9_]+)\.db$/i ? $1 : () } @all_tables;
}
foreach my $record (@all_tables) {
next unless $record;
next if $record =~ /^_/; # skip internal / temp files
if ( my ( $dbs, $tbl ) = ( $record =~ /^([a-z0-9]+)_(.+)$/i ) ) {
$all_tables{$dbs}->{$record} = 1;
if ( $schema_dir && !-e "$schema_dir/$dbs.dbase" ) {
$all_tables{__NO_DBASE__}->{$dbs} = 1;
}
if ( $schema_dir && !-e "$schema_dir/$record.table" ) {
$all_tables{__NO_TABLE__}->{$record} = 1;
}
}
else {
# Single-token table name without underscore
$all_tables{_main}->{$record} = 1;
if ( $schema_dir && !-e "$schema_dir/$record.table" ) {
$all_tables{__NO_TABLE__}->{$record} = 1;
}
}
}
if (wantarray) {
my @flat_list;
foreach my $dbs ( sort keys %all_tables ) {
next if $dbs =~ /^__/;
push @flat_list, sort keys %{ $all_tables{$dbs} };
}
return @flat_list;
}
return \%all_tables;
}
# =====================================================================
# TABLE MIGRATION & HISTORICAL FORMAT CONVERSION ENGINE
# =====================================================================
# Helper to binary copy a file safely without external modules
sub _copy_file {
lib/AmberDB/Tools/Maintain.pm view on Meta::CPAN
return 0 unless $k =~ /^[0-9]+$/;
return 0 unless ( ( defined $sfx && $sfx eq 'aut' ) ? $k >= 0 : $k > 0 );
return 1;
};
}
}
sub dump {
my ( $self, @args ) = @_;
my $adb = $self->{_adb} or return;
require Archive::Tar;
require Digest::SHA;
require JSON::PP;
require MIME::Base64;
require File::Path;
my %opts;
if ( @args == 1 && ref( $args[0] ) eq 'HASH' ) {
%opts = %{ $args[0] };
}
elsif ( @args % 2 == 1 && !ref( $args[0] ) ) {
my $first = shift @args;
%opts = @args;
if ( defined $first && length $first ) {
if ( $first =~ /\.(?:amberdb|tar\.gz|tgz)$/i || $first =~ m{[/\\]} ) {
$opts{file} //= $first;
}
else {
$opts{table} //= $first;
}
}
}
else {
%opts = @args;
}
# 1. Determine target tables (all tables if not specified)
my @tables;
if ( $opts{tables} && ref( $opts{tables} ) eq 'ARRAY' ) {
@tables = @{ $opts{tables} };
}
elsif ( $opts{table} ) {
@tables = ( $opts{table} );
}
else {
@tables = $self->all_tables();
}
return unless @tables;
# 2. Flush and close open table handles for read consistency
$adb->close_all();
# 3. Determine output file path
my $year = $adb->year;
my $month = $adb->month;
my $day = $adb->day;
my $date_iso = "$year-$month-$day";
my $time_id = $adb->second_id;
my $backup_base = $adb->path('backup_dir')
|| ( $adb->path('dbase_dir') ? $adb->path('dbase_dir') . "/backup" : "backup" );
my $year_dir = "$backup_base/$year";
$adb->make_path($year_dir);
my $outfile = $opts{file} || "$year_dir/amberdb_${date_iso}_${time_id}.amberdb";
# Ensure parent directory for $outfile exists
if ( my ($outdir) = $outfile =~ m{^(.*)[/\\]} ) {
$adb->make_path($outdir);
}
my $tar = Archive::Tar->new();
my $manifest = {
format => "AmberDB Archive",
format_version => 2,
amberdb_version => $AmberDB::VERSION || $VERSION,
created_at => "$date_iso " . sprintf( "%02d:%02d:%02d", (localtime)[2], (localtime)[1], (localtime)[0] ),
dbase_dir => $adb->path('dbase_dir'),
tables => {},
};
my $schema_dir = $adb->path('schema_dir')
|| ( $adb->path('dbase_dir') ? $adb->path('dbase_dir') . "/schema" : "schema" );
# Collect .dbase database group schemas
if ( -d $schema_dir ) {
opendir( my $sdh, $schema_dir );
my @dbase_files = grep { /\.dbase$/i } readdir($sdh);
closedir $sdh;
foreach my $df (@dbase_files) {
my ($dbs) = $df =~ /^([^.]+)\.dbase$/i;
next unless $dbs;
# If dumping specific tables, only include matching dbase prefix
if ( $opts{tables} || $opts{table} ) {
my $matched = 0;
foreach my $tid (@tables) {
if ( $tid =~ /^\Q$dbs\E_/ or $tid eq $dbs ) {
$matched = 1;
last;
}
}
next unless $matched;
}
my $dpath = "$schema_dir/$df";
if ( -e $dpath ) {
open my $dfh, "<:raw", $dpath or next;
local $/ = undef;
my $dcontent = <$dfh>;
close $dfh;
$tar->add_data( "schema/$df", $dcontent );
$manifest->{dbases}->{$dbs} = "schema/$df";
}
}
}
my $base_dir = $adb->path('dbase_dir') || ".";
$base_dir =~ s{\\}{/}g;
$base_dir =~ s{/$}{};
lib/AmberDB/Tools/Maintain.pm view on Meta::CPAN
my $fpath = "$tpath.$sfx";
if ( -e $fpath ) {
next unless $adb->table_read($fpath);
my $dcontent = "";
my $valid_keys = 0;
$adb->recs_scan(
$fpath,
sub {
my ( $k, $v ) = @_;
return unless $key_validator->( $k, $sfx );
my $b64 = MIME::Base64::encode_base64( $v, "" );
$dcontent .= "$k\t$b64\n";
$valid_keys++;
}
);
$adb->table_close($fpath);
my $dump_sfx = $sfx_map{$sfx};
my $norm_fpath = $fpath;
$norm_fpath =~ s{\\}{/}g;
my $arch_path;
if ( $norm_fpath =~ m{^\Q$base_dir\E/(.+)\.\Q$sfx\E$} ) {
$arch_path = "$1.$dump_sfx";
}
else {
$arch_path = "table/$tid.$dump_sfx";
}
$tar->add_data( $arch_path, $dcontent );
push @{ $table_manifest->{files} }, $arch_path;
my $sha256 = Digest::SHA::sha256_hex($dcontent);
$table_manifest->{sha256}->{$arch_path} = $sha256;
if ( $sfx eq $db_ext ) {
$table_manifest->{records} = $valid_keys;
}
}
}
$manifest->{tables}->{$tid} = $table_manifest;
}
# Add manifest.json to archive
my $json = JSON::PP->new->utf8->pretty->encode($manifest);
$tar->add_data( "manifest.json", $json );
# Write tar.gz archive
unless ( $tar->write( $outfile, Archive::Tar::COMPRESS_GZIP() ) ) {
cluck "[DB_BACKUP] Failed to write archive $outfile: " . $tar->error() . "\n";
return;
}
$self->{say} .= "Archive successfully written to $outfile (" . ( -s $outfile ) . " bytes)\n";
return wantarray ? ( $outfile, $manifest ) : $outfile;
}
# Restores a .amberdb archive into target database, validates checksums,
# writes data files via native DB_File, and deterministically rebuilds all indexes.
# my $res = $tools->restore( file => 'backup.amberdb', [force => 1], [reindex => 1] );
# ---------------------------------------------------------------------
sub restore {
my ( $self, @args ) = @_;
my $adb = $self->{_adb} or return;
my %opts;
if ( @args == 1 && ref( $args[0] ) eq 'HASH' ) {
%opts = %{ $args[0] };
}
elsif ( @args % 2 == 1 && !ref( $args[0] ) ) {
my $first = shift @args;
%opts = @args;
$opts{file} //= $first if defined $first && length $first;
}
else {
%opts = @args;
}
my $file = $opts{file} or do {
cluck "[DB_RESTORE] Missing required parameter 'file'.\n";
return;
};
return unless -e $file;
require Archive::Tar;
require Digest::SHA;
require JSON::PP;
require MIME::Base64;
require File::Path;
my $tar = Archive::Tar->new();
unless ( $tar->read($file) ) {
cluck "[DB_RESTORE] Cannot read archive $file: " . $tar->error() . "\n";
return;
}
# 1. Read and parse manifest.json
my $manifest_content = $tar->get_content("manifest.json");
unless ($manifest_content) {
cluck "[DB_RESTORE] Archive $file is missing manifest.json.\n";
return;
}
my $manifest = eval { JSON::PP->new->utf8->decode($manifest_content) };
if ( $@ || ref($manifest) ne 'HASH' ) {
cluck "[DB_RESTORE] Corrupted manifest.json in $file: $@\n";
return;
}
# 2. Check if target DB is empty or force is set
my $schema_dir = $adb->path('schema_dir')
|| ( $adb->path('dbase_dir') ? $adb->path('dbase_dir') . "/schema" : "schema" );
my $table_dir = $adb->path('table_dir')
|| ( $adb->path('dbase_dir') ? $adb->path('dbase_dir') . "/table" : "table" );
my $force = $opts{force} || $opts{overwrite};
unless ($force) {
my $has_existing = 0;
my $db_ext = $adb->{db_ext} || "db";
foreach my $tid ( keys %{ $manifest->{tables} || {} } ) {
( run in 1.362 second using v1.01-cache-2.11-cpan-99fa8243a42 )