Data-RecordStore

 view release on metacpan or  search on metacpan

lib/Data/RecordStore.pm  view on Meta::CPAN

    my $index = $self->[INDEX_SILO];
    $self->_write_lock;

    $index->ensure_entry_count( $id );
    if( defined $id && $id < 1 ) {
        die "The id must be a supplied as a positive integer";
    }
    my( $old_silo_id, $old_id_in_silo );
    if( $id > 0 ) {
        ( $old_silo_id, $old_id_in_silo ) = @{$index->get_record($id)};
    }
    else {
        $id = $index->next_id;
    }

    my $data_write_size = do { use bytes; length $_[1] };
    my $new_silo_id = $self->silo_id_for_size( $data_write_size );
    my $new_silo = $self->[SILOS][$new_silo_id];

    my $new_id_in_silo = $new_silo->push( [RS_ACTIVE, $id, $data_write_size, $_[1]] );

    $index->put_record( $id, [$new_silo_id,$new_id_in_silo,time] );

    if( $old_silo_id ) {
        $self->_vacate( $old_silo_id, $old_id_in_silo );
    }

    $self->_unlock;
    return $id;
} #stow

sub next_id {
    return shift->[INDEX_SILO]->next_id;
} #next_id

sub delete_record {
    my( $self, $del_id ) = @_;
    $self->_write_lock;
    my $trans = $self->[TRANSACTION];
    if( $trans ) {
        $self->_unlock;
        return $trans->delete_record( $del_id );
    }

    if( $del_id > $self->[INDEX_SILO]->entry_count ) {
        warn "Tried to delete past end of records";
        $self->_unlock;
        return undef;
    }
    my( $old_silo_id, $old_id_in_silo ) = @{$self->[INDEX_SILO]->get_record($del_id)};
    $self->[INDEX_SILO]->put_record( $del_id, [0,0,time] );

    if( $old_silo_id ) {
        $self->_vacate( $old_silo_id, $old_id_in_silo );
    }
    $self->_unlock;
} #delete_record

# locks the given lock names
# they are locked in order to prevent deadlocks.
sub lock {
    my( $self, @locknames ) = @_;

    my( %previously_locked ) = ( map { $_ => 1 } @{$self->[LOCKS]} );

    if( @{$self->[LOCKS]} && grep { ! $previously_locked{$_} } @locknames ) {
        die "Data::RecordStore->lock cannot be called twice in a row without unlocking between";
    }
    my $fhs = [];

    my $failed;

    for my $name (sort @locknames) {
        next if $previously_locked{$name}++;
        my $lockfile = "$self->[DIRECTORY]/user_locks/$name";
        my $fh;
        if( -e $lockfile ) {
            unless( open ( $fh, '+<', $lockfile ) ) {
                $failed = 1;
                last;
            }
            flock( $fh, LOCK_EX ); #WRITE LOCK
        }
        else {
            unless( open( $fh, '>', $lockfile ) ) {
                $failed = 1;
                last;
            }
            flock( $fh, LOCK_EX ); #WRITE LOCK
            $fh->autoflush(1);
            print $fh '';
        }
        push @$fhs, $fh;
    }

    if( $failed ) {
        # it must be able to lock all the locks or it fails
        # if it failed, unlock any locks it managed to get
        for my $fh (@$fhs) {
            flock( $fh, LOCK_UN );
        }
        die "Data::RecordStore->lock : lock failed";
    } else {
        $self->[LOCKS] = $fhs;
    }

} #lock

# unlocks all locks
sub unlock {
    my $self = shift;
    my $fhs = $self->[LOCKS];

    for my $fh (@$fhs) {
        flock( $fh, LOCK_UN );
    }
    @$fhs = ();
} #unlock

sub use_transaction {
    my $self = shift;
    if( $self->[TRANSACTION] ) {
        warn __PACKAGE__."->use_transaction : already in transaction";
        return $self->[TRANSACTION];
    }
    $self->_write_lock;
    my $tid = $self->[TRANSACTION_INDEX_SILO]->push( [TR_ACTIVE, time] );
    $self->_unlock;
    my $tdir = "$self->[DIRECTORY]/transactions/$tid";
    make_path( $tdir, { error => \my $err } );
    if( @$err ) { die join( ", ", map { values %$_ } @$err ) }

    $self->[TRANSACTION] = Data::RecordStore::Transaction->create( $self, $tdir, $tid );
    return $self->[TRANSACTION];

lib/Data/RecordStore.pm  view on Meta::CPAN

        while( $rc > $id_to_empty ) {
            my( $state, $id ) = (@{$silo->get_record( $rc, 'IL' )});
            if( $state == RS_ACTIVE ) {
                $silo->copy_record($rc,$id_to_empty);
                $self->[INDEX_SILO]->put_record( $id, [$silo_id,$id_to_empty], "IL" );
                $silo->pop;
                return;
            }
            elsif( $state == RS_DEAD ) {
                $silo->pop;
            }
            else {
                return;
            }
            $rc--;
        }
    }
} #_vacate

sub silo_id_for_size {
    my( $self, $data_write_size ) = @_;

    my $write_size = $self->[HEADER_SIZE] + $data_write_size;

    my $silo_id = int( log( $write_size ) / log( 2 ) );
    $silo_id++ if 2 ** $silo_id < $write_size;
    $silo_id = $self->[MIN_SILO_ID] if $silo_id < $self->[MIN_SILO_ID];
    return $silo_id;
} #silo_id_for_size

# ---------------------- private stuffs -------------------------

sub _make_path {
    my( $dir, $msg ) = @_;
    make_path( $dir, { error => \my $err } );
    if( @$err ) {
        die "unable to make $msg directory.". join( ", ", map { $_->{$dir} } @$err );
    }
}

sub _read_lock {
    my $self = shift;
    flock( $self->[LOCK_FH], LOCK_SH );
    $self->_fix_transactions;
}

sub _unlock {
    my( $self ) = @_;
    flock( $self->[LOCK_FH], LOCK_UN );
}

sub _write_lock {
    my $self = shift;
    flock( $self->[LOCK_FH], LOCK_EX );
    $self->_fix_transactions;
}

sub _fix_transactions {
    my $self = shift;
    # check the transactions
    # if the transaction is in an incomplete state, fix it. Since the store is write locked
    # during transactions, the lock has expired if this point has been reached.
    # that means the process that made the lock has fallen.
    #
    # of course, do a little experiment to test this with two processes and flock when
    # one exits before unflocking.
    #
    my $transaction_index_silo = $self->transaction_silo;
    my $last_trans = $transaction_index_silo->entry_count;
    while( $last_trans ) {
        my( $state ) = @{$transaction_index_silo->get_record( $last_trans )};
        my $tdir = "$self->[DIRECTORY]/transactions/$last_trans";
        if( $state == TR_IN_ROLLBACK ||
                $state == TR_IN_COMMIT ) {
            # do a full rollback
            # load the transaction
            my $trans = Data::RecordStore::Transaction->create( $self, $tdir, $last_trans );
            $trans->rollback;
            $transaction_index_silo->pop;
        }
        elsif( $state == TR_COMPLETE ) {
            $transaction_index_silo->pop;
        }
        else {
            return;
        }
        $last_trans--;
    }

} #_fix_transactions


"I became Iggy because I had a sadistic boss at a record store. I'd been in a band called the Iguanas. And when this boss wanted to embarrass and demean me, he'd say, 'Iggy, get me a coffee, light.' - Iggy Pop";

__END__

=head1 NAME

 Data::RecordStore - Simple store for text and byte data

=head1 SYNPOSIS

 use Data::RecordStore;

 $store = Data::RecordStore->init_store( DIRECTORY => $directory, MAX_FILE_SIZE => 20_000_000_000 );
 $data = "TEXT OR BYTES";

 # the first record id is 1
 my $id = $store->stow( $data );

 my $val = $store->fetch( $some_id );

 my $count = $store->entry_count;

 $store->lock( qw( FEE FIE FOE FUM ) ); # lock blocks, may not be called until unlock.

 $store->unlock; # unlocks all

 $store->delete_record( $id_to_remove ); #deletes the old record

 $reopened_store = Data::RecordStore->open_store( $directory );

lib/Data/RecordStore.pm  view on Meta::CPAN

Incomplete transactions are obtained by the store's 'list_transactions'
method.

Data::RecordStore operates directly and instantly on the file system.
It is not a daemon or server and is not thread safe. It can be used
in a thread safe manner if the controlling program uses locking mechanisms,
including the locks that the store provides.

=head1 METHODS

=head2 open_store( options )

Constructs a data store according to the options.

Options

=over 2

=item BASE_PATH

=item MIN_FILE_SIZE - default is 4096

=item MAX_FILE_SIZE - default is 2 gigs

=head2 reopen_store( directory )

Opens the existing store in the given directory.

=head2 fetch( id )

Returns the record associated with the ID. If the ID has no
record associated with it, undef is returned.

=head2 stow( data, optionalID )

This saves the text or byte data to the record store.
If an id is passed in, this saves the data to the record
for that id, overwriting what was there.
If an id is not passed in, it creates a new record store.

Returns the id of the record written to.

=head2 next_id

This sets up a new empty record and returns the
id for it.

=head2 delete_record( id )

Removes the entry with the given id from the store, freeing up its space.
It does not reuse the id.

=head2 lock( @names )

Adds an advisory (flock) lock for each of the unique names given.
This may not be called twice in a row without an unlock in between
and will die if that happens.

=head2 unlock

Unlocks all names locked by this thread

=head2 use_transaction()

Returns the current transaction. If there is no
current transaction, it creates one and returns it.

=head2 commit_transaction()

Commits the current transaction, if any.

=head2 rollback_transaction()

Rolls back the current transaction, if any.

=head2 entry_count

Returns how many record ids exist.

=head2 index_silo

Returns the index silo for this store. 
This method is not part of the record store interface.

=head2 max_file_size

Returns the max file size of any silo in bytes.
This method is not part of the record store interface.

=head2 silos

Returns a list of data silo objects where the data silo record
size is 2**index position. This means that the beginning of the list
will have undefs as there is a minimum silo size.
This method is not part of the record store interface.

=head2 transaction_silo

Returns the transaction silo for this store.
This method is not part of the record store interface.

=head2 active_entry_count

Returns how many record ids exist that have silo entries.
This method is not part of the record store interface.

=head2 silos_entry_count

Returns the number of entries in the data silos.

=head2 detect_version( $dir )

Tries to detect the version of the Data::RecordStore in
the given directory, if any.

=head1 AUTHOR
       Eric Wolf        coyocanid@gmail.com

=head1 COPYRIGHT AND LICENSE

       Copyright (c) 2015-2020 Eric Wolf. All rights reserved.