view release on metacpan or  search on metacpan

lib/Data/NDArray/Shared.pm  view on Meta::CPAN

sub _require_pdl {
    # require PDL alone does NOT define the type-constructor subs (&PDL::double,
    # &PDL::long, ...) nor new_from_specification -- those live in PDL::Core, which
    # `use PDL` loads but `require PDL` does not.  Load it explicitly so to_pdl works.
    eval { require PDL; require PDL::Core; 1 }
        or Carp::croak("Data::NDArray::Shared: PDL interop needs PDL installed (cpanm PDL)");
}
sub _pdl_ctor {
    my ($dtype) = @_;
    my $name = $PDL_TYPE{$dtype} or Carp::croak("no PDL type for dtype '$dtype'");
    exists &{"PDL::$name"}
        or Carp::croak("this PDL has no '$name' type (needed for dtype '$dtype'); upgrade PDL");
    \&{"PDL::$name"};
}

# NDArray -> a NEW (copied) PDL piddle; dims = reverse(shape).
sub to_pdl {
    my ($self) = @_;
    _require_pdl();
    my $p = PDL->new_from_specification(_pdl_ctor($self->dtype)->(), reverse $self->shape);
    ${ $p->get_dataref } = $self->buffer;   # read-locked snapshot
    $p->upd_data;
    return $p;
}

# A NEW shared NDArray copied from a piddle; $path undef => anonymous mapping.
sub from_pdl {
    my ($class, $p, $path) = @_;
    _require_pdl();
    my $tname = "" . $p->type;
    my $dt = $DTYPE_OF{$tname}
        or Carp::croak("Data::NDArray::Shared->from_pdl: unsupported PDL type '$tname'");
    $p = $p->copy;                          # force a contiguous, physical piddle
    my $self = $class->new($path, $dt, reverse $p->dims);
    $self->update_from_bytes(${ $p->get_dataref });
    return $self;
}

# Copy a piddle into THIS array in place (same dtype + shape); returns self.
sub update_from_pdl {
    my ($self, $p) = @_;
    _require_pdl();
    my $tname = "" . $p->type;
    my $dt = $DTYPE_OF{$tname}
        or Carp::croak("Data::NDArray::Shared->update_from_pdl: unsupported PDL type '$tname'");
    $dt eq $self->dtype
        or Carp::croak("update_from_pdl: dtype mismatch (piddle $dt vs array " . $self->dtype . ")");
    my @want = reverse $self->shape;
    my @got  = $p->dims;
    "@want" eq "@got"
        or Carp::croak("update_from_pdl: shape mismatch (array (@{[ $self->shape ]}) vs piddle dims (@got))");
    $p = $p->copy;
    $self->update_from_bytes(${ $p->get_dataref });
    return $self;
}

# Zero-copy: a PDL ndarray ALIASING this array's shared mmap, built via PDL's C
# API (PDL_DONTTOUCHDATA, so PDL never frees/reallocates our mapping).  In-place
# PDL ops write straight through (visible to every sharing process); reads see
# live data.  NO locking -- coordinate access yourself.  The array is kept alive
# while the piddle lives.  Needs PDL at BUILD time (the C path); croaks otherwise.
sub as_pdl_alias {
    my ($self) = @_;
    _require_pdl();
    my $typenum = _pdl_ctor($self->dtype)->()->enum;        # PDL type number for our dtype
    # _alias_pdl_create croaks if the module was built without PDL (no C path).
    my $p = $self->_alias_pdl_create($typenum, [ reverse $self->shape ]);   # dims in PDL order
    $p->hdr->{_nda_shared} = $self;   # keep the mapping alive while the piddle lives
    return $p;
}

1;
__END__

=encoding utf-8

=head1 NAME

Data::NDArray::Shared - shared-memory typed N-dimensional numeric array for Linux

=head1 SYNOPSIS

    use Data::NDArray::Shared;

    # a 2x3 array of doubles in an anonymous shared mapping
    # ($path = undef for an anonymous array)
    my $a = Data::NDArray::Shared->new(undef, "f64", 2, 3);

    $a->ndim;        # 2
    $a->size;        # 6   (== 2 * 3, also ->numel)
    $a->shape;       # (2, 3)
    $a->strides;     # (3, 1)   row-major, in elements
    $a->dtype;       # "f64"
    $a->itemsize;    # 8

    $a->set(0, 0, 1.5);          # element [0][0] = 1.5  (multi-index)
    $a->get(0, 0);               # 1.5
    $a->set_flat(5, 9);          # last element by flat index
    $a->get_flat(5);             # 9

    $a->fill(7);                 # every element = 7
    $a->zero;                    # every element = 0

    $a->sum;  $a->mean;  $a->min;  $a->max;   # whole-array reductions

    $a->add_scalar(2);           # every element += 2   (in place)
    $a->mul_scalar(3);           # every element *= 3   (in place)

    $a->reshape(3, 2);           # same data, shape (3,2), strides (2,1)

    # element-wise array arithmetic (same dtype + total size), in place
    my $b = Data::NDArray::Shared->new(undef, "f64", 3, 2);
    $a->add($b);                 # a[i] += b[i]
    $a->subtract($b);            # a[i] -= b[i]
    $a->multiply($b);            # a[i] *= b[i]

    my $list = $a->to_list;      # arrayref of all elements, row-major

    # integer dtypes: i64/i32/i16/i8/u64/u32/u16/u8
    my $c = Data::NDArray::Shared->new(undef, "u8", 4);
    $c->set_flat(0, 300);        # wraps to 44 (stored in the element width)

    # share across processes via a backing file ($path = the file)
    my $shared = Data::NDArray::Shared->new("/tmp/nd.bin", "f64", 100, 100);

    # freeze and ship: query it read-only (lock-free) on other machines
    $shared->freeze;
    my $ro = Data::NDArray::Shared->new_readonly("/tmp/nd.bin");

lib/Data/NDArray/Shared.pm  view on Meta::CPAN

=item * C<itemsize> -- bytes per element.

=item * C<shape> -- an arrayref of the dimension sizes.

=item * C<ops> -- running count of operations that took the write lock (every
C<set>, C<set_flat>, C<fill>, C<zero>, C<reshape>, C<add_scalar>,
C<mul_scalar>, C<add>, C<subtract>, C<multiply>).

=item * C<mmap_size> -- bytes of the shared mapping.

=item * C<frozen> -- 1 if the array has been sealed by C<freeze> (immutable), else 0.

=item * C<readonly> -- 1 if this handle is a read-only view (from C<new_readonly>,
or the handle that called C<freeze>), else 0.

=back

=head1 PDL INTEROP

If L<PDL> is installed the array converts to and from PDL ndarrays. PDL is an
B<optional, load-on-demand> dependency -- there is no build- or runtime prereq;
the four conversion methods (C<to_pdl>, C<from_pdl>, C<update_from_pdl>,
C<as_pdl_alias>) C<croak> if PDL is missing, while C<buffer> and
C<update_from_bytes> have no PDL dependency. Each dtype maps to a PDL type of
the B<same byte width> (C<f64> to C<double>, C<i32> to C<long>, C<u64> to
C<ulonglong>, and so on), so the data moves with no per-element conversion.

B<Axis order:> this array is row-major (C-order) while PDL's C<dim(0)> is the
B<fastest-varying> axis, so the shape is B<reversed> across the boundary -- an
C<($r, $c)> array corresponds to PDL dims C<($c, $r)>, and
C<< $piddle-E<gt>at($j, $i) >> is C<< $array-E<gt>get($i, $j) >>. The conversion
methods handle this for you.

=over 4

=item * C<< $piddle = $array->to_pdl >>

A B<new> piddle holding a B<copy> of the data, of the mapped PDL type and dims
C<< reverse($array-E<gt>shape) >>. Read under the lock, so it is a consistent
snapshot.

=item * C<< $array = Data::NDArray::Shared->from_pdl($piddle, $path) >>

A B<new> shared array B<copied> from C<$piddle> (made physical and contiguous
first); the dtype and shape follow the piddle's type and C<reverse> of its dims.
C<$path> is the backing file (C<undef> or omitted for an anonymous mapping).

=item * C<< $array->update_from_pdl($piddle) >>

Copy C<$piddle> into this array B<in place> (write-locked). The piddle's type
must match the dtype and its dims must equal C<< reverse($array-E<gt>shape) >>,
else it croaks. Returns the array.

=item * C<< $piddle = $array->as_pdl_alias >>

A piddle that B<aliases the shared mapping with no copy> (a real
C<PDL_DONTTOUCHDATA> ndarray over our memory): an B<in-place> PDL operation
(C<< $p .= ... >>, C<< $p-E<gt>inplace-E<gt>... >>) writes straight through to
shared memory -- visible to every process that maps it -- and reads see live
data. The array is kept alive for as long as the piddle.

This one method needs PDL at B<build> time (it is compiled against PDL's C API):
if the module was installed without PDL present it C<croak>s, while the copy
methods above keep working through a runtime C<require PDL>. Reinstall with PDL
installed to enable it.

B<Caveats.> The alias B<bypasses the rwlock>: you must coordinate access
yourself (no other process mutating concurrently), as with any unlocked
shared-memory view. Do not B<resize or retype> the alias (a reshape that grows
it, a type conversion) -- it is a fixed window onto the mapping; use
C<to_pdl>/C<from_pdl> when you want an independent, resizable copy.

On a B<frozen> array (see L</"FROZEN (READ-ONLY) MODE">) the mapping itself is
C<PROT_READ>, and PDL has no write path that is reliably safe against that (its
own read-only flag stops C<< .= >> and in-place ops, but not
C<< $piddle->set(...) >>, which pokes the buffer directly). So C<as_pdl_alias>
B<refuses to alias a frozen array> and C<croak>s instead of handing back a
piddle that could still crash the process; use C<to_pdl> for a safe copy.

=item * C<< $bytes = $array->buffer >>

The raw contiguous data region as a byte string (read-locked snapshot),
row-major C-order -- useful on its own for serialization or IPC, and the basis
for C<to_pdl>. C<< $array->update_from_bytes($bytes) >> is the inverse
(write-locked; the string must be exactly C<< size * itemsize >> bytes).

=back

See F<eg/pdl_interop.pl> for a worked example, including a cross-process PDL
transform on one shared array.

=head1 FROZEN (READ-ONLY) MODE

A file-backed array can be B<frozen> and then shipped to other machines, where
consumers open it B<read-only> and query it with B<no locking at all>.

    # producer: build, freeze, ship the file
    my $a = Data::NDArray::Shared->new("/tmp/weights.bin", "f64", 100, 100);
    $a->fill(0); $a->set(0, 0, 1.5);   # ... populate ...
    $a->freeze;                  # seal: now immutable, and $a itself is read-only
    # ... copy /tmp/weights.bin to another host ...

    # consumer (any process, same architecture): read-only, lock-free
    my $ro = Data::NDArray::Shared->new_readonly("/tmp/weights.bin");
    $ro->get(0, 0);
    $ro->sum;

C<freeze> takes the write lock, marks the array B<permanently immutable> (there
is no unfreeze -- rebuild the file to change it), and flushes the seal to disk.
A frozen array rejects every mutator (C<set>, C<set_flat>, C<fill>, C<zero>,
C<reshape>, C<add_scalar>, C<mul_scalar>, C<add>, C<subtract>, C<multiply>,
C<update_from_bytes>) with a croak, and a read-write reopen (C<< new($path,
...) >>) of a sealed file is B<refused> -- so a shipped artifact can never be
silently mutated out from under its readers.

C<new_readonly($path)> maps the file C<O_RDONLY> / C<PROT_READ> and B<requires
it to be frozen> (it croaks on a file that was never C<freeze>d). Because a
sealed array's data, dtype, shape and strides are all immutable (C<reshape> is
one of the refused mutators), every read -- C<get>, C<get_flat>, C<sum>,
C<mean>, C<min>, C<max>, C<to_list>, C<shape>, C<strides>, C<stats>, C<buffer>