AmberDB

 view release on metacpan or  search on metacpan

lib/AmberDB/Tools/Maintain.pm  view on Meta::CPAN


    my ( $self, $tableid ) = @_;
    my $adb = $self->{_adb} or return;

    $tableid or return;
    my $table_path = $adb->table_path($tableid);

    my $i = 1;
    if ( -e "$table_path.csv" ) {
        my $day_id = $adb->day_id;
        rename( "$table_path.csv", "$table_path-$day_id.csv" );
    }

    # cevirme islemlerini yap
    my $tie_path = "${table_path}.$adb->{db_ext}";
    return 1 unless -e $tie_path;
    $adb->table_read($tie_path) or return 1;

    my @records;
    $adb->recs_scan(
        $tie_path,
        sub {
            my ( $k, $v ) = @_;
            my @fields = $adb->db_decode( $v );
            push @records, [ $k, $adb->tsv_encode( @fields ) ];
            $self->{say} .= "$i. $k ID record converted.\n\n";
            $i++;
        }
    );

    @records = sort { $a->[0] <=> $b->[0] } @records;
    open my $fh, ">", "$table_path.csv" or do {
        cluck "[DB_TOOL] Could not open $table_path.csv: $!\n";
        return;
    };
    foreach my $record ( @records ) {
        print $fh "$record->[0]\t$record->[1]\n";
    }
    close $fh;
    $adb->table_close($tie_path);

    return 1;
}

# $tools->{ISO2UTF} = 1;
# my $ok = $tools->csv2tie("file");
# ---------------------------------------------------------------------
sub csv2tie {

    my ( $self, $tableid ) = @_;
    my $adb = $self->{_adb} or return;

    $tableid or return;

    # file paths
    my $table_path = $adb->table_path($tableid);
    my $file_path  = "${table_path}.$adb->{db_ext}";
    my $csv_path   = "$table_path.csv";
    return unless -e $csv_path;

    # backup with timestamp if exists
    if ( -e "${table_path}.$adb->{db_ext}" ) {
        my $sec_id = $adb->second_id;
        rename( "${table_path}.$adb->{db_ext}",
            "${table_path}-$sec_id.$adb->{db_ext}" );
        unlink("${table_path}.$adb->{db_ext}");
    }

    # perform conversion operations
    my $i = 1;
    my @records;

    open my $FH, "<", $csv_path or do {
        cluck "[DB_TOOL] Could not open $csv_path: $!\n";
        return;
    };
    $adb->table_write($file_path) or do {
        close $FH;
        cluck "[DB_TOOL] Could not open $file_path for writing.\n";
        return;
    };
    my @chunk;
    while ( my $record = <$FH> ) {
        chomp($record);
        $record =~ s/\r$//;
        my (@fields) = $adb->db_decode($record);
        push @chunk, [@fields];
        if ( @chunk >= 2000 ) {
            $adb->recs_put( $file_path, @chunk );
            @chunk = ();
        }

        # collect into list for indexing
        push @records, [@fields];
        $self->{say} .= "$i. Record ID $fields[0] converted.\n\n" if $i % 1000 == 0;
        $i++;
    }
    if ( @chunk ) {
        $adb->recs_put( $file_path, @chunk );
    }
    close $FH;
    $adb->table_close($file_path);

    $self->set_index( $tableid, @records );

    return 1;
}

# my @tables = $tools->dir_tables();
# ------------------------------------------------
sub dir_tables {

    my ( $self, $dir ) = @_;
    my $adb = $self->{_adb} or return;

    $dir or return;
    my $target_dir = File::Spec->catdir( $adb->path('dbase_dir') || ".", $dir );
    my $ext        = $adb->{db_ext} || "db";
    my @names      = $adb->dir_files( $target_dir, "*.$ext", full_path => 0 );

    my %all_tables = map { /^(.+)\.\Q$ext\E$/i ? ( $1 => 1 ) : () } @names;

lib/AmberDB/Tools/Maintain.pm  view on Meta::CPAN

    # Reverse operation
    rename( $tie_path, $tie_back );

    open $FH, "<", $csv_path or do {
        cluck "[DB_TOOL] Could not open $csv_path for reading: $!\n";
        return;
    };
    $adb->table_write($tie_path);
    my @chunk;
    while ( my $record = <$FH> ) {
        chomp($record);
        $record =~ s/\r$//;
        my @fields = $adb->db_decode($record);
        push @chunk, [@fields];
        if ( @chunk >= 2000 ) {
            my $ok = $adb->recs_put( $tie_path, @chunk );
            $count->{tie2} += scalar @chunk if $ok;
            @chunk = ();
        }
    }
    if ( @chunk ) {
        my $ok = $adb->recs_put( $tie_path, @chunk );
        $count->{tie2} += scalar @chunk if $ok;
    }
    close $FH;
    $adb->table_close($tie_path);

    $adb->table_attr( $tableid, $table_info );
    $self->set_index( $tableid, @records ) if $reindex;
    $self->{say} .= "Vacuum completed. Record counts: Tie old: $count->{tie1}, CSV: $count->{csv1}, Tie new: $count->{tie2}\n";

    return 1;
}

# my $tables_hash = $tools->all_tables(); # scalar context -> grouped hashref
# my @table_list  = $tools->all_tables(); # list context   -> flat array of table IDs
# ------------------------------------------------
sub all_tables {

    my ($self) = @_;
    my $adb = $self->{_adb} or return;

    my @all_tables;
    my %all_tables;

    my $dbase_dir  = $adb->path('dbase_dir')  || ".";
    my $year_dir   = $adb->path('year_dir')   || "";
    my $schema_dir = $adb->path('schema_dir') || "";

    # 1. Simple Mode: Single flat directory scan for files matching configured db_ext
    if ( $adb->config('use_simple') ) {
        my $ext = $adb->{db_ext} || "db";
        my @files = $adb->dir_files( $dbase_dir, "*.$ext", full_path => 0 );
        @all_tables = map { /^([a-z0-9_]+)\.\Q$ext\E$/i ? $1 : () } @files;
    }
    # 2. Standard Structured Mode: Multi-directory scan (table/ and year directories) for .db files
    else {
        my $tbl_dir = File::Spec->catdir( $dbase_dir, 'table' );
        push @all_tables, $adb->dir_files( $tbl_dir, "*.db", full_path => 0 ) if -d $tbl_dir;

        my %seen_dirs = ( "table" => 1, "schema" => 1, "backup" => 1, "lock" => 1, "ramdisk" => 1, "journal" => 1, "session" => 1, "config" => 1 );
        if ($year_dir) {
            my $yd_path = File::Spec->catdir( $dbase_dir, $year_dir );
            push @all_tables, $adb->dir_files( $yd_path, "*.db", full_path => 0 );
            $seen_dirs{$year_dir} = 1;
        }

        # Auto-discover any 4-digit year directories under $dbase_dir (e.g. 2024, 2025, 2026)
        if ( -d $dbase_dir ) {
            my @year_dirs = $adb->dir_files( $dbase_dir, qr/^\d{4}$/, full_path => 0, files_only => 0 );
            foreach my $yd (@year_dirs) {
                next if $seen_dirs{$yd} || !-d File::Spec->catdir( $dbase_dir, $yd );
                my $yd_path = File::Spec->catdir( $dbase_dir, $yd );
                push @all_tables, $adb->dir_files( $yd_path, "*.db", full_path => 0 );
            }
        }

        @all_tables = map { /^([a-z0-9_]+)\.db$/i ? $1 : () } @all_tables;
    }

    foreach my $record (@all_tables) {
        next unless $record;
        next if $record =~ /^_/; # skip internal / temp files

        if ( my ( $dbs, $tbl ) = ( $record =~ /^([a-z0-9]+)_(.+)$/i ) ) {
            $all_tables{$dbs}->{$record} = 1;
            if ( $schema_dir && !-e "$schema_dir/$dbs.dbase" ) {
                $all_tables{__NO_DBASE__}->{$dbs} = 1;
            }
            if ( $schema_dir && !-e "$schema_dir/$record.table" ) {
                $all_tables{__NO_TABLE__}->{$record} = 1;
            }
        }
        else {
            # Single-token table name without underscore
            $all_tables{_main}->{$record} = 1;
            if ( $schema_dir && !-e "$schema_dir/$record.table" ) {
                $all_tables{__NO_TABLE__}->{$record} = 1;
            }
        }
    }

    if (wantarray) {
        my @flat_list;
        foreach my $dbs ( sort keys %all_tables ) {
            next if $dbs =~ /^__/;
            push @flat_list, sort keys %{ $all_tables{$dbs} };
        }
        return @flat_list;
    }

    return \%all_tables;
}

# =====================================================================
# TABLE MIGRATION & HISTORICAL FORMAT CONVERSION ENGINE
# =====================================================================


# Helper to binary copy a file safely without external modules
sub _copy_file {

lib/AmberDB/Tools/Maintain.pm  view on Meta::CPAN

            return 0 unless $k =~ /^[0-9]+$/;
            return 0 unless ( ( defined $sfx && $sfx eq 'aut' ) ? $k >= 0 : $k > 0 );
            return 1;
        };
    }
}

sub dump {
    my ( $self, @args ) = @_;
    my $adb = $self->{_adb} or return;

    require Archive::Tar;
    require Digest::SHA;
    require JSON::PP;
    require MIME::Base64;
    require File::Path;

    my %opts;
    if ( @args == 1 && ref( $args[0] ) eq 'HASH' ) {
        %opts = %{ $args[0] };
    }
    elsif ( @args % 2 == 1 && !ref( $args[0] ) ) {
        my $first = shift @args;
        %opts = @args;
        if ( defined $first && length $first ) {
            if ( $first =~ /\.(?:amberdb|tar\.gz|tgz)$/i || $first =~ m{[/\\]} ) {
                $opts{file} //= $first;
            }
            else {
                $opts{table} //= $first;
            }
        }
    }
    else {
        %opts = @args;
    }

    # 1. Determine target tables (all tables if not specified)
    my @tables;
    if ( $opts{tables} && ref( $opts{tables} ) eq 'ARRAY' ) {
        @tables = @{ $opts{tables} };
    }
    elsif ( $opts{table} ) {
        @tables = ( $opts{table} );
    }
    else {
        @tables = $self->all_tables();
    }
    return unless @tables;

    # 2. Flush and close open table handles for read consistency
    $adb->close_all();

    # 3. Determine output file path
    my $year     = $adb->year;
    my $month    = $adb->month;
    my $day      = $adb->day;
    my $date_iso = "$year-$month-$day";
    my $time_id  = $adb->second_id;

    my $backup_base = $adb->path('backup_dir')
      || ( $adb->path('dbase_dir') ? $adb->path('dbase_dir') . "/backup" : "backup" );
    my $year_dir = "$backup_base/$year";
    $adb->make_path($year_dir);

    my $outfile = $opts{file} || "$year_dir/amberdb_${date_iso}_${time_id}.amberdb";

    # Ensure parent directory for $outfile exists
    if ( my ($outdir) = $outfile =~ m{^(.*)[/\\]} ) {
        $adb->make_path($outdir);
    }

    my $tar = Archive::Tar->new();

    my $manifest = {
        format          => "AmberDB Archive",
        format_version  => 2,
        amberdb_version => $AmberDB::VERSION || $VERSION,
        created_at      => "$date_iso " . sprintf( "%02d:%02d:%02d", (localtime)[2], (localtime)[1], (localtime)[0] ),
        dbase_dir       => $adb->path('dbase_dir'),
        tables          => {},
    };

    my $schema_dir = $adb->path('schema_dir')
      || ( $adb->path('dbase_dir') ? $adb->path('dbase_dir') . "/schema" : "schema" );

    # Collect .dbase database group schemas
    if ( -d $schema_dir ) {
        opendir( my $sdh, $schema_dir );
        my @dbase_files = grep { /\.dbase$/i } readdir($sdh);
        closedir $sdh;

        foreach my $df (@dbase_files) {
            my ($dbs) = $df =~ /^([^.]+)\.dbase$/i;
            next unless $dbs;

            # If dumping specific tables, only include matching dbase prefix
            if ( $opts{tables} || $opts{table} ) {
                my $matched = 0;
                foreach my $tid (@tables) {
                    if ( $tid =~ /^\Q$dbs\E_/ or $tid eq $dbs ) {
                        $matched = 1;
                        last;
                    }
                }
                next unless $matched;
            }

            my $dpath = "$schema_dir/$df";
            if ( -e $dpath ) {
                open my $dfh, "<:raw", $dpath or next;
                local $/ = undef;
                my $dcontent = <$dfh>;
                close $dfh;
                $tar->add_data( "schema/$df", $dcontent );
                $manifest->{dbases}->{$dbs} = "schema/$df";
            }
        }
    }

    my $base_dir = $adb->path('dbase_dir') || ".";
    $base_dir =~ s{\\}{/}g;
    $base_dir =~ s{/$}{};

lib/AmberDB/Tools/Maintain.pm  view on Meta::CPAN

            my $fpath = "$tpath.$sfx";
            if ( -e $fpath ) {
                next unless $adb->table_read($fpath);

                my $dcontent   = "";
                my $valid_keys = 0;
                $adb->recs_scan(
                    $fpath,
                    sub {
                        my ( $k, $v ) = @_;
                        return unless $key_validator->( $k, $sfx );
                        my $b64 = MIME::Base64::encode_base64( $v, "" );
                        $dcontent .= "$k\t$b64\n";
                        $valid_keys++;
                    }
                );
                $adb->table_close($fpath);

                my $dump_sfx = $sfx_map{$sfx};
                my $norm_fpath = $fpath;
                $norm_fpath =~ s{\\}{/}g;
                my $arch_path;
                if ( $norm_fpath =~ m{^\Q$base_dir\E/(.+)\.\Q$sfx\E$} ) {
                    $arch_path = "$1.$dump_sfx";
                }
                else {
                    $arch_path = "table/$tid.$dump_sfx";
                }

                $tar->add_data( $arch_path, $dcontent );
                push @{ $table_manifest->{files} }, $arch_path;

                my $sha256 = Digest::SHA::sha256_hex($dcontent);
                $table_manifest->{sha256}->{$arch_path} = $sha256;

                if ( $sfx eq $db_ext ) {
                    $table_manifest->{records} = $valid_keys;
                }
            }
        }

        $manifest->{tables}->{$tid} = $table_manifest;
    }

    # Add manifest.json to archive
    my $json = JSON::PP->new->utf8->pretty->encode($manifest);
    $tar->add_data( "manifest.json", $json );

    # Write tar.gz archive
    unless ( $tar->write( $outfile, Archive::Tar::COMPRESS_GZIP() ) ) {
        cluck "[DB_BACKUP] Failed to write archive $outfile: " . $tar->error() . "\n";
        return;
    }

    $self->{say} .= "Archive successfully written to $outfile (" . ( -s $outfile ) . " bytes)\n";
    return wantarray ? ( $outfile, $manifest ) : $outfile;
}

# Restores a .amberdb archive into target database, validates checksums,
# writes data files via native DB_File, and deterministically rebuilds all indexes.
# my $res = $tools->restore( file => 'backup.amberdb', [force => 1], [reindex => 1] );
# ---------------------------------------------------------------------
sub restore {
    my ( $self, @args ) = @_;
    my $adb = $self->{_adb} or return;

    my %opts;
    if ( @args == 1 && ref( $args[0] ) eq 'HASH' ) {
        %opts = %{ $args[0] };
    }
    elsif ( @args % 2 == 1 && !ref( $args[0] ) ) {
        my $first = shift @args;
        %opts = @args;
        $opts{file} //= $first if defined $first && length $first;
    }
    else {
        %opts = @args;
    }

    my $file = $opts{file} or do {
        cluck "[DB_RESTORE] Missing required parameter 'file'.\n";
        return;
    };
    return unless -e $file;

    require Archive::Tar;
    require Digest::SHA;
    require JSON::PP;
    require MIME::Base64;
    require File::Path;

    my $tar = Archive::Tar->new();
    unless ( $tar->read($file) ) {
        cluck "[DB_RESTORE] Cannot read archive $file: " . $tar->error() . "\n";
        return;
    }

    # 1. Read and parse manifest.json
    my $manifest_content = $tar->get_content("manifest.json");
    unless ($manifest_content) {
        cluck "[DB_RESTORE] Archive $file is missing manifest.json.\n";
        return;
    }

    my $manifest = eval { JSON::PP->new->utf8->decode($manifest_content) };
    if ( $@ || ref($manifest) ne 'HASH' ) {
        cluck "[DB_RESTORE] Corrupted manifest.json in $file: $@\n";
        return;
    }

    # 2. Check if target DB is empty or force is set
    my $schema_dir = $adb->path('schema_dir')
      || ( $adb->path('dbase_dir') ? $adb->path('dbase_dir') . "/schema" : "schema" );
    my $table_dir = $adb->path('table_dir')
      || ( $adb->path('dbase_dir') ? $adb->path('dbase_dir') . "/table" : "table" );

    my $force = $opts{force} || $opts{overwrite};
    unless ($force) {
        my $has_existing = 0;
        my $db_ext = $adb->{db_ext} || "db";
        foreach my $tid ( keys %{ $manifest->{tables} || {} } ) {



( run in 1.362 second using v1.01-cache-2.11-cpan-99fa8243a42 )