AI-MXNet
view release on metacpan or search on metacpan
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
package AI::MXNet::RNN::Params;
use Mouse;
use AI::MXNet::Function::Parameters;
=head1 NAME
AI::MXNet::RNN::Params
=cut
=head1 DESCRIPTION
A container for holding variables.
Used by RNN cells for parameter sharing between cells.
Parameters
----------
prefix : str
All variables name created by this container will
be prepended with the prefix
=cut
has '_prefix' => (is => 'ro', init_arg => 'prefix', isa => 'Str', default => '');
has '_params' => (is => 'rw', init_arg => undef);
around BUILDARGS => sub {
my $orig = shift;
my $class = shift;
return $class->$orig(prefix => $_[0]) if @_ == 1;
return $class->$orig(@_);
};
sub BUILD
{
my $self = shift;
$self->_params({});
}
=head2 get
Get a variable with the name or create a new one if does not exist.
Parameters
----------
$name : str
name of the variable
@kwargs:
more arguments that are passed to mx->sym->Variable call
=cut
method get(Str $name, @kwargs)
{
$name = $self->_prefix . $name;
if(not exists $self->_params->{$name})
{
$self->_params->{$name} = AI::MXNet::Symbol->Variable($name, @kwargs);
}
return $self->_params->{$name};
}
package AI::MXNet::RNN::Cell::Base;
=head1 NAME
AI::MXNet::RNNCell::Base
=cut
=head1 DESCRIPTION
Abstract base class for RNN cells
Parameters
----------
prefix : str
prefix for name of layers
(and name of weight if params is undef)
params : AI::MXNet::RNN::Params or undef
container for weight sharing between cells.
created if undef.
=cut
use AI::MXNet::Base;
use Mouse;
use overload "&{}" => sub { my $self = shift; sub { $self->call(@_) } };
has '_prefix' => (is => 'rw', init_arg => 'prefix', isa => 'Str', default => '');
has '_params' => (is => 'rw', init_arg => 'params', isa => 'Maybe[AI::MXNet::RNN::Params]');
has [qw/_own_params
_modified
_init_counter
_counter
/] => (is => 'rw', init_arg => undef);
around BUILDARGS => sub {
my $orig = shift;
my $class = shift;
return $class->$orig(prefix => $_[0]) if @_ == 1;
return $class->$orig(@_);
};
sub BUILD
{
my $self = shift;
if(not defined $self->_params)
{
$self->_own_params(1);
$self->_params(AI::MXNet::RNN::Params->new($self->_prefix));
}
else
{
$self->_own_params(0);
}
$self->_modified(0);
$self->reset;
}
=head2 reset
Reset before re-using the cell for another graph
=cut
method reset()
{
$self->_init_counter(-1);
$self->_counter(-1);
}
=head2 call
Construct symbol for one step of RNN.
Parameters
----------
$inputs : mx->sym->Variable
input symbol, 2D, batch * num_units
$states : mx->sym->Variable or ArrayRef[AI::MXNet::Symbol]
state from previous step or begin_state().
Returns
-------
$output : AI::MXNet::Symbol
output symbol
$states : ArrayRef[AI::MXNet::Symbol]
state to next step of RNN.
Can be called via overloaded &{}: &{$cell}($inputs, $states);
=cut
method call(AI::MXNet::Symbol $inputs, AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol] $states)
{
confess("Not Implemented");
}
method _gate_names()
{
[''];
}
=head2 params
Parameters of this cell
=cut
method params()
{
$self->_own_params(0);
return $self->_params;
}
=head2 state_shape
shape(s) of states
=cut
method state_shape()
{
return [map { $_->{shape} } @{ $self->state_info }];
}
=head2 state_info
shape and layout information of states
=cut
method state_info()
{
confess("Not Implemented");
}
=head2 begin_state
Initial state for this cell.
Parameters
----------
:$func : sub ref, default is AI::MXNet::Symbol->can('zeros')
Function for creating initial state.
Can be AI::MXNet::Symbol->can('zeros'),
AI::MXNet::Symbol->can('uniform'), AI::MXNet::Symbol->can('Variable') etc.
Use AI::MXNet::Symbol->can('Variable') if you want to directly
feed the input as states.
@kwargs :
more keyword arguments passed to func. For example
mean, std, dtype, etc.
Returns
-------
$states : ArrayRef[AI::MXNet::Symbol]
starting states for first RNN step
=cut
method begin_state(CodeRef :$func=AI::MXNet::Symbol->can('zeros'), @kwargs)
{
assert(
(not $self->_modified),
"After applying modifier cells (e.g. DropoutCell) the base "
."cell cannot be called directly. Call the modifier cell instead."
);
my @states;
my $func_needs_named_name = $func ne AI::MXNet::Symbol->can('Variable');
for my $info (@{ $self->state_info })
{
$self->_init_counter($self->_init_counter + 1);
my @name = (sprintf("%sbegin_state_%d", $self->_prefix, $self->_init_counter));
my %info = %{ $info//{} };
if($func_needs_named_name)
{
unshift(@name, 'name');
}
else
{
if(exists $info{__layout__})
{
$info{kwargs} = { __layout__ => delete $info{__layout__} };
}
}
my %kwargs = (@kwargs, %info);
my $state = &{$func}(
'AI::MXNet::Symbol',
@name,
%kwargs
);
push @states, $state;
}
return \@states;
}
=head2 unpack_weights
Unpack fused weight matrices into separate
weight matrices
Parameters
----------
$args : HashRef[AI::MXNet::NDArray]
hash ref containing packed weights.
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
}
return($outputs, $states);
}
method _get_activation($inputs, $activation, @kwargs)
{
if(not ref $activation)
{
return AI::MXNet::Symbol->Activation($inputs, act_type => $activation, @kwargs);
}
else
{
return &{$activation}($inputs, @kwargs);
}
}
method _cells_state_shape($cells)
{
return [map { @{ $_->state_shape } } @$cells];
}
method _cells_state_info($cells)
{
return [map { @{ $_->state_info } } @$cells];
}
method _cells_begin_state($cells, @kwargs)
{
return [map { @{ $_->begin_state(@kwargs) } } @$cells];
}
method _cells_unpack_weights($cells, $args)
{
$args = $_->unpack_weights($args) for @$cells;
return $args;
}
method _cells_pack_weights($cells, $args)
{
$args = $_->pack_weights($args) for @$cells;
return $args;
}
package AI::MXNet::RNN::Cell;
use Mouse;
extends 'AI::MXNet::RNN::Cell::Base';
=head1 NAME
AI::MXNet::RNN::Cell
=cut
=head1 DESCRIPTION
Simple recurrent neural network cell
Parameters
----------
num_hidden : int
number of units in output symbol
activation : str or Symbol, default 'tanh'
type of activation function
prefix : str, default 'rnn_'
prefix for name of layers
(and name of weight if params is undef)
params : AI::MXNet::RNNParams or undef
container for weight sharing between cells.
created if undef.
=cut
has '_num_hidden' => (is => 'ro', init_arg => 'num_hidden', isa => 'Int', required => 1);
has 'forget_bias' => (is => 'ro', isa => 'Num');
has '_activation' => (
is => 'ro',
init_arg => 'activation',
isa => 'Activation',
default => 'tanh'
);
has '+_prefix' => (default => 'rnn_');
has [qw/_iW _iB
_hW _hB/] => (is => 'rw', init_arg => undef);
around BUILDARGS => sub {
my $orig = shift;
my $class = shift;
return $class->$orig(num_hidden => $_[0]) if @_ == 1;
return $class->$orig(@_);
};
sub BUILD
{
my $self = shift;
$self->_iW($self->params->get('i2h_weight'));
$self->_iB(
$self->params->get(
'i2h_bias',
(defined($self->forget_bias)
? (init => AI::MXNet::LSTMBias->new(forget_bias => $self->forget_bias))
: ()
)
)
);
$self->_hW($self->params->get('h2h_weight'));
$self->_hB($self->params->get('h2h_bias'));
}
method state_info()
{
return [{ shape => [0, $self->_num_hidden], __layout__ => 'NC' }];
}
method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states)
{
$self->_counter($self->_counter + 1);
my $name = sprintf('%st%d_', $self->_prefix, $self->_counter);
my $i2h = AI::MXNet::Symbol->FullyConnected(
data => $inputs,
weight => $self->_iW,
bias => $self->_iB,
num_hidden => $self->_num_hidden,
name => "${name}i2h"
);
my $h2h = AI::MXNet::Symbol->FullyConnected(
data => @{$states}[0],
weight => $self->_hW,
bias => $self->_hB,
num_hidden => $self->_num_hidden,
name => "${name}h2h"
);
my $output = $self->_get_activation(
$i2h + $h2h,
$self->_activation,
name => "${name}out"
);
return ($output, [$output]);
}
package AI::MXNet::RNN::LSTMCell;
use Mouse;
use AI::MXNet::Base;
extends 'AI::MXNet::RNN::Cell';
=head1 NAME
AI::MXNet::RNN::LSTMCell
=cut
=head1 DESCRIPTION
Long-Short Term Memory (LSTM) network cell.
Parameters
----------
num_hidden : int
number of units in output symbol
prefix : str, default 'lstm_'
prefix for name of layers
(and name of weight if params is undef)
params : AI::MXNet::RNN::Params or None
container for weight sharing between cells.
created if undef.
forget_bias : bias added to forget gate, default 1.0.
Jozefowicz et al. 2015 recommends setting this to 1.0
=cut
has '+_prefix' => (default => 'lstm_');
has '+_activation' => (init_arg => undef);
has '+forget_bias' => (is => 'ro', isa => 'Num', default => 1);
method state_info()
{
return [{ shape => [0, $self->_num_hidden], __layout__ => 'NC' } , { shape => [0, $self->_num_hidden], __layout__ => 'NC' }];
}
method _gate_names()
{
[qw/_i _f _c _o/];
}
method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states)
{
$self->_counter($self->_counter + 1);
my $name = sprintf('%st%d_', $self->_prefix, $self->_counter);
my @states = @{ $states };
my $i2h = AI::MXNet::Symbol->FullyConnected(
data => $inputs,
weight => $self->_iW,
bias => $self->_iB,
num_hidden => $self->_num_hidden*4,
name => "${name}i2h"
);
my $h2h = AI::MXNet::Symbol->FullyConnected(
data => $states[0],
weight => $self->_hW,
bias => $self->_hB,
num_hidden => $self->_num_hidden*4,
name => "${name}h2h"
);
my $gates = $i2h + $h2h;
my @slice_gates = @{ AI::MXNet::Symbol->SliceChannel(
$gates, num_outputs => 4, name => "${name}slice"
) };
my $in_gate = AI::MXNet::Symbol->Activation(
$slice_gates[0], act_type => "sigmoid", name => "${name}i"
);
my $forget_gate = AI::MXNet::Symbol->Activation(
$slice_gates[1], act_type => "sigmoid", name => "${name}f"
);
my $in_transform = AI::MXNet::Symbol->Activation(
$slice_gates[2], act_type => "tanh", name => "${name}c"
);
my $out_gate = AI::MXNet::Symbol->Activation(
$slice_gates[3], act_type => "sigmoid", name => "${name}o"
);
my $next_c = AI::MXNet::Symbol->_plus(
$forget_gate * $states[1], $in_gate * $in_transform,
name => "${name}state"
);
my $next_h = AI::MXNet::Symbol->_mul(
$out_gate,
AI::MXNet::Symbol->Activation(
$next_c, act_type => "tanh"
),
name => "${name}out"
);
return ($next_h, [$next_h, $next_c]);
}
package AI::MXNet::RNN::GRUCell;
use Mouse;
use AI::MXNet::Base;
extends 'AI::MXNet::RNN::Cell';
=head1 NAME
AI::MXNet::RNN::GRUCell
=cut
=head1 DESCRIPTION
Gated Rectified Unit (GRU) network cell.
Note: this is an implementation of the cuDNN version of GRUs
(slight modification compared to Cho et al. 2014).
Parameters
----------
num_hidden : int
number of units in output symbol
prefix : str, default 'gru_'
prefix for name of layers
(and name of weight if params is undef)
params : AI::MXNet::RNN::Params or undef
container for weight sharing between cells.
created if undef.
=cut
has '+_prefix' => (default => 'gru_');
method _gate_names()
{
[qw/_r _z _o/];
}
method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states)
{
$self->_counter($self->_counter + 1);
my $name = sprintf('%st%d_', $self->_prefix, $self->_counter);
my $prev_state_h = @{ $states }[0];
my $i2h = AI::MXNet::Symbol->FullyConnected(
data => $inputs,
weight => $self->_iW,
bias => $self->_iB,
num_hidden => $self->_num_hidden*3,
name => "${name}i2h"
);
my $h2h = AI::MXNet::Symbol->FullyConnected(
data => $prev_state_h,
weight => $self->_hW,
bias => $self->_hB,
num_hidden => $self->_num_hidden*3,
name => "${name}h2h"
);
my ($i2h_r, $i2h_z);
($i2h_r, $i2h_z, $i2h) = @{ AI::MXNet::Symbol->SliceChannel(
$i2h, num_outputs => 3, name => "${name}_i2h_slice"
) };
my ($h2h_r, $h2h_z);
($h2h_r, $h2h_z, $h2h) = @{ AI::MXNet::Symbol->SliceChannel(
$h2h, num_outputs => 3, name => "${name}_h2h_slice"
) };
my $reset_gate = AI::MXNet::Symbol->Activation(
$i2h_r + $h2h_r, act_type => "sigmoid", name => "${name}_r_act"
);
my $update_gate = AI::MXNet::Symbol->Activation(
$i2h_z + $h2h_z, act_type => "sigmoid", name => "${name}_z_act"
);
my $next_h_tmp = AI::MXNet::Symbol->Activation(
$i2h + $reset_gate * $h2h, act_type => "tanh", name => "${name}_h_act"
);
my $next_h = AI::MXNet::Symbol->_plus(
(1 - $update_gate) * $next_h_tmp, $update_gate * $prev_state_h,
name => "${name}out"
);
return ($next_h, [$next_h]);
}
package AI::MXNet::RNN::FusedCell;
use Mouse;
use AI::MXNet::Types;
use AI::MXNet::Base;
extends 'AI::MXNet::RNN::Cell::Base';
=head1 NAME
AI::MXNet::RNN::FusedCell
=cut
=head1 DESCRIPTION
Fusing RNN layers across time step into one kernel.
Improves speed but is less flexible. Currently only
supported if using cuDNN on GPU.
=cut
has '_num_hidden' => (is => 'ro', isa => 'Int', init_arg => 'num_hidden', required => 1);
has '_num_layers' => (is => 'ro', isa => 'Int', init_arg => 'num_layers', default => 1);
has '_dropout' => (is => 'ro', isa => 'Num', init_arg => 'dropout', default => 0);
has '_get_next_state' => (is => 'ro', isa => 'Bool', init_arg => 'get_next_state', default => 0);
has '_bidirectional' => (is => 'ro', isa => 'Bool', init_arg => 'bidirectional', default => 0);
has 'forget_bias' => (is => 'ro', isa => 'Num', default => 1);
has 'initializer' => (is => 'rw', isa => 'Maybe[Initializer]');
has '_mode' => (
is => 'ro',
isa => enum([qw/rnn_relu rnn_tanh lstm gru/]),
init_arg => 'mode',
default => 'lstm'
);
has [qw/_parameter
_directions/] => (is => 'rw', init_arg => undef);
around BUILDARGS => sub {
my $orig = shift;
my $class = shift;
return $class->$orig(num_hidden => $_[0]) if @_ == 1;
return $class->$orig(@_);
};
sub BUILD
{
my $self = shift;
if(not $self->_prefix)
{
$self->_prefix($self->_mode.'_');
}
if(not defined $self->initializer)
{
$self->initializer(
AI::MXNet::Xavier->new(
factor_type => 'in',
magnitude => 2.34
)
);
}
if(not $self->initializer->isa('AI::MXNet::FusedRNN'))
{
$self->initializer(
AI::MXNet::FusedRNN->new(
init => $self->initializer,
num_hidden => $self->_num_hidden,
num_layers => $self->_num_layers,
mode => $self->_mode,
bidirectional => $self->_bidirectional,
forget_bias => $self->forget_bias
)
);
}
$self->_parameter($self->params->get('parameters', init => $self->initializer));
$self->_directions($self->_bidirectional ? [qw/l r/] : ['l']);
}
method state_info()
{
my $b = @{ $self->_directions };
my $n = $self->_mode eq 'lstm' ? 2 : 1;
return [map { +{ shape => [$b*$self->_num_layers, 0, $self->_num_hidden], __layout__ => 'LNC' } } 0..$n-1];
}
method _gate_names()
{
return {
rnn_relu => [''],
rnn_tanh => [''],
lstm => [qw/_i _f _c _o/],
gru => [qw/_r _z _o/]
}->{ $self->_mode };
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
my $state = [@{ $states }[$p..$p+$n-1]];
$p += $n;
($inputs, $state) = &{$cell}($inputs, $state);
push @next_states, $state;
}
return ($inputs, [map { @$_} @next_states]);
}
method unroll(
Int $length,
Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$inputs=,
Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$begin_state=,
Str :$input_prefix='',
Str :$layout='NTC',
Maybe[Bool] :$merge_outputs=
)
{
my $num_cells = @{ $self->_cells };
$begin_state //= $self->begin_state;
my $p = 0;
my $states;
my @next_states;
enumerate(sub {
my ($i, $cell) = @_;
my $n = @{ $cell->state_info };
$states = [@{$begin_state}[$p..$p+$n-1]];
$p += $n;
($inputs, $states) = $cell->unroll(
$length,
inputs => $inputs,
input_prefix => $input_prefix,
begin_state => $states,
layout => $layout,
merge_outputs => ($i < $num_cells-1) ? undef : $merge_outputs
);
push @next_states, $states;
}, $self->_cells);
return ($inputs, [map { @{ $_ } } @next_states]);
}
package AI::MXNet::RNN::BidirectionalCell;
use Mouse;
use AI::MXNet::Base;
extends 'AI::MXNet::RNN::Cell::Base';
=head1 NAME
AI::MXNet::RNN::BidirectionalCell
=cut
=head1 DESCRIPTION
Bidirectional RNN cell
Parameters
----------
l_cell : AI::MXNet::RNN::Cell::Base
cell for forward unrolling
r_cell : AI::MXNet::RNN::Cell::Base
cell for backward unrolling
output_prefix : str, default 'bi_'
prefix for name of output
=cut
has 'l_cell' => (is => 'ro', isa => 'AI::MXNet::RNN::Cell::Base', required => 1);
has 'r_cell' => (is => 'ro', isa => 'AI::MXNet::RNN::Cell::Base', required => 1);
has '_output_prefix' => (is => 'ro', init_arg => 'output_prefix', isa => 'Str', default => 'bi_');
has [qw/_override_cell_params _cells/] => (is => 'rw', init_arg => undef);
around BUILDARGS => sub {
my $orig = shift;
my $class = shift;
if(@_ >= 2 and blessed $_[0] and blessed $_[1])
{
my $l_cell = shift(@_);
my $r_cell = shift(@_);
return $class->$orig(
l_cell => $l_cell,
r_cell => $r_cell,
@_
);
}
return $class->$orig(@_);
};
sub BUILD
{
my ($self, $original_arguments) = @_;
$self->_override_cell_params(defined $original_arguments->{params});
if($self->_override_cell_params)
{
assert(
($self->l_cell->_own_params and $self->r_cell->_own_params),
"Either specify params for BidirectionalCell ".
"or child cells, not both."
);
%{ $self->l_cell->params->_params } = (%{ $self->l_cell->params->_params }, %{ $self->params->_params });
%{ $self->r_cell->params->_params } = (%{ $self->r_cell->params->_params }, %{ $self->params->_params });
}
%{ $self->params->_params } = (%{ $self->params->_params }, %{ $self->l_cell->params->_params });
%{ $self->params->_params } = (%{ $self->params->_params }, %{ $self->r_cell->params->_params });
$self->_cells([$self->l_cell, $self->r_cell]);
}
method unpack_weights(HashRef[AI::MXNet::NDArray] $args)
{
return $self->_cells_unpack_weights($self->_cells, $args)
}
method pack_weights(HashRef[AI::MXNet::NDArray] $args)
{
return $self->_cells_pack_weights($self->_cells, $args);
}
method call($inputs, $states)
{
confess("Bidirectional cannot be stepped. Please use unroll");
}
method state_info()
{
return $self->_cells_state_info($self->_cells);
}
method begin_state(@kwargs)
{
assert((not $self->_modified),
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
$r_outputs, axis => $axis,
num_outputs => $length,
squeeze_axis => 1
) }
];
}
}
}
if($merge_outputs)
{
$l_outputs = [@{ $l_outputs }];
$r_outputs = [@{ AI::MXNet::Symbol->reverse(blessed $r_outputs ? $r_outputs : @{ $r_outputs }, axis=>$axis) }];
}
else
{
$r_outputs = [reverse(@{ $r_outputs })];
}
my $outputs = [];
zip(sub {
my ($i, $l_o, $r_o) = @_;
push @$outputs, AI::MXNet::Symbol->Concat(
$l_o, $r_o, dim=>(1+($merge_outputs?1:0)),
name => $merge_outputs
? sprintf('%sout', $self->_output_prefix)
: sprintf('%st%d', $self->_output_prefix, $i)
);
}, [0..@{ $l_outputs }-1], [@{ $l_outputs }], [@{ $r_outputs }]);
if($merge_outputs)
{
$outputs = @{ $outputs }[0];
}
$states = [$l_states, $r_states];
return($outputs, $states);
}
package AI::MXNet::RNN::ConvCell::Base;
use Mouse;
use AI::MXNet::Base;
extends 'AI::MXNet::RNN::Cell::Base';
=head1 NAME
AI::MXNet::RNN::Conv::Base
=cut
=head1 DESCRIPTION
Abstract base class for Convolutional RNN cells
=cut
has '_h2h_kernel' => (is => 'ro', isa => 'Shape', init_arg => 'h2h_kernel');
has '_h2h_dilate' => (is => 'ro', isa => 'Shape', init_arg => 'h2h_dilate');
has '_h2h_pad' => (is => 'rw', isa => 'Shape', init_arg => undef);
has '_i2h_kernel' => (is => 'ro', isa => 'Shape', init_arg => 'i2h_kernel');
has '_i2h_stride' => (is => 'ro', isa => 'Shape', init_arg => 'i2h_stride');
has '_i2h_dilate' => (is => 'ro', isa => 'Shape', init_arg => 'i2h_dilate');
has '_i2h_pad' => (is => 'ro', isa => 'Shape', init_arg => 'i2h_pad');
has '_num_hidden' => (is => 'ro', isa => 'DimSize', init_arg => 'num_hidden');
has '_input_shape' => (is => 'ro', isa => 'Shape', init_arg => 'input_shape');
has '_conv_layout' => (is => 'ro', isa => 'Str', init_arg => 'conv_layout', default => 'NCHW');
has '_activation' => (is => 'ro', init_arg => 'activation');
has '_state_shape' => (is => 'rw', init_arg => undef);
has [qw/i2h_weight_initializer h2h_weight_initializer
i2h_bias_initializer h2h_bias_initializer/] => (is => 'rw', isa => 'Maybe[Initializer]');
sub BUILD
{
my $self = shift;
assert (
($self->_h2h_kernel->[0] % 2 == 1 and $self->_h2h_kernel->[1] % 2 == 1),
"Only support odd numbers, got h2h_kernel= (@{[ $self->_h2h_kernel ]})"
);
$self->_h2h_pad([
int($self->_h2h_dilate->[0] * ($self->_h2h_kernel->[0] - 1) / 2),
int($self->_h2h_dilate->[1] * ($self->_h2h_kernel->[1] - 1) / 2)
]);
# Infer state shape
my $data = AI::MXNet::Symbol->Variable('data');
my $state_shape = AI::MXNet::Symbol->Convolution(
data => $data,
num_filter => $self->_num_hidden,
kernel => $self->_i2h_kernel,
stride => $self->_i2h_stride,
pad => $self->_i2h_pad,
dilate => $self->_i2h_dilate,
layout => $self->_conv_layout
);
$state_shape = ($state_shape->infer_shape(data=>$self->_input_shape))[1]->[0];
$state_shape->[0] = 0;
$self->_state_shape($state_shape);
}
method state_info()
{
return [
{ shape => $self->_state_shape, __layout__ => $self->_conv_layout },
{ shape => $self->_state_shape, __layout__ => $self->_conv_layout }
];
}
method call($inputs, $states)
{
confess("AI::MXNet::RNN::ConvCell::Base is abstract class for convolutional RNN");
}
package AI::MXNet::RNN::ConvCell;
use Mouse;
extends 'AI::MXNet::RNN::ConvCell::Base';
=head1 NAME
AI::MXNet::RNN::ConvCell
=cut
=head1 DESCRIPTION
Convolutional RNN cells
Parameters
----------
input_shape : array ref of int
Shape of input in single timestep.
num_hidden : int
Number of units in output symbol.
h2h_kernel : array ref of int, default (3, 3)
Kernel of Convolution operator in state-to-state transitions.
h2h_dilate : array ref of int, default (1, 1)
Dilation of Convolution operator in state-to-state transitions.
i2h_kernel : array ref of int, default (3, 3)
Kernel of Convolution operator in input-to-state transitions.
i2h_stride : array ref of int, default (1, 1)
Stride of Convolution operator in input-to-state transitions.
i2h_pad : array ref of int, default (1, 1)
Pad of Convolution operator in input-to-state transitions.
i2h_dilate : array ref of int, default (1, 1)
Dilation of Convolution operator in input-to-state transitions.
activation : str or Symbol,
default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2)
Type of activation function.
prefix : str, default 'ConvRNN_'
Prefix for name of layers (and name of weight if params is None).
params : RNNParams, default None
Container for weight sharing between cells. Created if None.
conv_layout : str, , default 'NCHW'
Layout of ConvolutionOp
=cut
has '+_h2h_kernel' => (default => sub { [3, 3] });
has '+_h2h_dilate' => (default => sub { [1, 1] });
has '+_i2h_kernel' => (default => sub { [3, 3] });
has '+_i2h_stride' => (default => sub { [1, 1] });
has '+_i2h_dilate' => (default => sub { [1, 1] });
has '+_i2h_pad' => (default => sub { [1, 1] });
has '+_prefix' => (default => 'ConvRNN_');
has '+_activation' => (default => sub { sub { AI::MXNet::Symbol->LeakyReLU(@_, act_type => 'leaky', slope => 0.2) } });
has '+i2h_bias_initializer' => (default => 'zeros');
has '+h2h_bias_initializer' => (default => 'zeros');
has 'forget_bias' => (is => 'ro', isa => 'Num');
has [qw/_iW _iB
_hW _hB/] => (is => 'rw', init_arg => undef);
sub BUILD
{
my $self = shift;
$self->_iW($self->_params->get('i2h_weight', init => $self->i2h_weight_initializer));
$self->_hW($self->_params->get('h2h_weight', init => $self->h2h_weight_initializer));
$self->_iB(
$self->params->get(
'i2h_bias',
(defined($self->forget_bias and not defined $self->i2h_bias_initializer)
? (init => AI::MXNet::LSTMBias->new(forget_bias => $self->forget_bias))
: (init => $self->i2h_bias_initializer)
)
)
);
$self->_hB($self->_params->get('h2h_bias', init => $self->h2h_bias_initializer));
}
method _num_gates()
{
scalar(@{ $self->_gate_names() });
}
method _gate_names()
{
return ['']
}
method _conv_forward($inputs, $states, $name)
{
my $i2h = AI::MXNet::Symbol->Convolution(
name => "${name}i2h",
data => $inputs,
num_filter => $self->_num_hidden*$self->_num_gates(),
kernel => $self->_i2h_kernel,
stride => $self->_i2h_stride,
pad => $self->_i2h_pad,
dilate => $self->_i2h_dilate,
weight => $self->_iW,
bias => $self->_iB
);
my $h2h = AI::MXNet::Symbol->Convolution(
name => "${name}h2h",
data => @{ $states }[0],
num_filter => $self->_num_hidden*$self->_num_gates(),
kernel => $self->_h2h_kernel,
stride => [1, 1],
pad => $self->_h2h_pad,
dilate => $self->_h2h_dilate,
weight => $self->_hW,
bias => $self->_hB
);
return ($i2h, $h2h);
}
method call(AI::MXNet::Symbol $inputs, AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol] $states)
{
$self->_counter($self->_counter + 1);
my $name = sprintf('%st%d_', $self->_prefix, $self->_counter);
my ($i2h, $h2h) = $self->_conv_forward($inputs, $states, $name);
my $output = $self->_get_activation($i2h + $h2h, $self->_activation, name => "${name}out");
return ($output, [$output]);
}
package AI::MXNet::RNN::ConvLSTMCell;
use Mouse;
extends 'AI::MXNet::RNN::ConvCell';
has '+forget_bias' => (default => 1);
has '+_prefix' => (default => 'ConvLSTM_');
=head1 NAME
AI::MXNet::RNN::ConvLSTMCell
=cut
=head1 DESCRIPTION
Convolutional LSTM network cell.
Reference:
Xingjian et al. NIPS2015
=cut
method _gate_names()
{
return ['_i', '_f', '_c', '_o'];
}
method call(AI::MXNet::Symbol $inputs, AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol] $states)
{
$self->_counter($self->_counter + 1);
my $name = sprintf('%st%d_', $self->_prefix, $self->_counter);
my ($i2h, $h2h) = $self->_conv_forward($inputs, $states, $name);
my $gates = $i2h + $h2h;
my @slice_gates = @{ AI::MXNet::Symbol->SliceChannel(
$gates,
num_outputs => 4,
axis => index($self->_conv_layout, 'C'),
name => "${name}slice"
) };
my $in_gate = AI::MXNet::Symbol->Activation(
$slice_gates[0],
act_type => "sigmoid",
name => "${name}i"
);
my $forget_gate = AI::MXNet::Symbol->Activation(
$slice_gates[1],
act_type => "sigmoid",
name => "${name}f"
);
my $in_transform = $self->_get_activation(
$slice_gates[2],
$self->_activation,
name => "${name}c"
);
my $out_gate = AI::MXNet::Symbol->Activation(
$slice_gates[3],
act_type => "sigmoid",
name => "${name}o"
);
my $next_c = AI::MXNet::Symbol->_plus(
$forget_gate * @{$states}[1],
$in_gate * $in_transform,
name => "${name}state"
);
my $next_h = AI::MXNet::Symbol->_mul(
$out_gate, $self->_get_activation($next_c, $self->_activation),
name => "${name}out"
);
return ($next_h, [$next_h, $next_c]);
}
package AI::MXNet::RNN::ConvGRUCell;
use Mouse;
extends 'AI::MXNet::RNN::ConvCell';
has '+_prefix' => (default => 'ConvGRU_');
=head1 NAME
AI::MXNet::RNN::ConvGRUCell
=cut
=head1 DESCRIPTION
Convolutional GRU network cell.
=cut
method _gate_names()
{
return ['_r', '_z', '_o'];
}
method call(AI::MXNet::Symbol $inputs, AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol] $states)
{
$self->_counter($self->_counter + 1);
my $name = sprintf('%st%d_', $self->_prefix, $self->_counter);
my ($i2h, $h2h) = $self->_conv_forward($inputs, $states, $name);
my ($i2h_r, $i2h_z, $h2h_r, $h2h_z);
($i2h_r, $i2h_z, $i2h) = @{ AI::MXNet::Symbol->SliceChannel($i2h, num_outputs => 3, name => "${name}_i2h_slice") };
($h2h_r, $h2h_z, $h2h) = @{ AI::MXNet::Symbol->SliceChannel($h2h, num_outputs => 3, name => "${name}_h2h_slice") };
my $reset_gate = AI::MXNet::Symbol->Activation(
$i2h_r + $h2h_r, act_type => "sigmoid",
name => "${name}_r_act"
);
my $update_gate = AI::MXNet::Symbol->Activation(
$i2h_z + $h2h_z, act_type => "sigmoid",
name => "${name}_z_act"
);
my $next_h_tmp = $self->_get_activation($i2h + $reset_gate * $h2h, $self->_activation, name => "${name}_h_act");
my $next_h = AI::MXNet::Symbol->_plus(
(1 - $update_gate) * $next_h_tmp, $update_gate * @{$states}[0],
name => "${name}out"
);
return ($next_h, [$next_h]);
}
package AI::MXNet::RNN::ModifierCell;
use Mouse;
use AI::MXNet::Base;
extends 'AI::MXNet::RNN::Cell::Base';
=head1 NAME
AI::MXNet::RNN::ModifierCell
=cut
=head1 DESCRIPTION
Base class for modifier cells. A modifier
cell takes a base cell, apply modifications
on it (e.g. Dropout), and returns a new cell.
After applying modifiers the base cell should
no longer be called directly. The modifer cell
should be used instead.
=cut
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
around BUILDARGS => sub {
my $orig = shift;
my $class = shift;
if(@_%2)
{
my $base_cell = shift;
return $class->$orig(base_cell => $base_cell, @_);
}
return $class->$orig(@_);
};
sub BUILD
{
my $self = shift;
$self->base_cell->_modified(1);
}
method params()
{
$self->_own_params(0);
return $self->base_cell->params;
}
method state_info()
{
return $self->base_cell->state_info;
}
method begin_state(CodeRef :$init_sym=AI::MXNet::Symbol->can('zeros'), @kwargs)
{
assert(
(not $self->_modified),
"After applying modifier cells (e.g. DropoutCell) the base "
."cell cannot be called directly. Call the modifier cell instead."
);
$self->base_cell->_modified(0);
my $begin_state = $self->base_cell->begin_state(func => $init_sym, @kwargs);
$self->base_cell->_modified(1);
return $begin_state;
}
method unpack_weights(HashRef[AI::MXNet::NDArray] $args)
{
return $self->base_cell->unpack_weights($args)
}
method pack_weights(HashRef[AI::MXNet::NDArray] $args)
{
return $self->base_cell->pack_weights($args)
}
method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states)
{
confess("Not Implemented");
}
package AI::MXNet::RNN::DropoutCell;
use Mouse;
extends 'AI::MXNet::RNN::ModifierCell';
has [qw/dropout_outputs dropout_states/] => (is => 'ro', isa => 'Num', default => 0);
=head1 NAME
AI::MXNet::RNN::DropoutCell
=cut
=head1 DESCRIPTION
Apply the dropout on base cell
=cut
method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states)
{
my ($output, $states) = &{$self->base_cell}($inputs, $states);
if($self->dropout_outputs > 0)
{
$output = AI::MXNet::Symbol->Dropout(data => $output, p => $self->dropout_outputs);
}
if($self->dropout_states > 0)
{
$states = [map { AI::MXNet::Symbol->Dropout(data => $_, p => $self->dropout_states) } @{ $states }];
}
return ($output, $states);
}
package AI::MXNet::RNN::ZoneoutCell;
use Mouse;
use AI::MXNet::Base;
extends 'AI::MXNet::RNN::ModifierCell';
has [qw/zoneout_outputs zoneout_states/] => (is => 'ro', isa => 'Num', default => 0);
has 'prev_output' => (is => 'rw', init_arg => undef);
=head1 NAME
AI::MXNet::RNN::ZoneoutCell
=cut
=head1 DESCRIPTION
Apply Zoneout on base cell.
=cut
sub BUILD
{
my $self = shift;
assert(
(not $self->base_cell->isa('AI::MXNet::RNN::FusedCell')),
"FusedRNNCell doesn't support zoneout. ".
"Please unfuse first."
);
assert(
(not $self->base_cell->isa('AI::MXNet::RNN::BidirectionalCell')),
"BidirectionalCell doesn't support zoneout since it doesn't support step. ".
"Please add ZoneoutCell to the cells underneath instead."
);
assert(
(not $self->base_cell->isa('AI::MXNet::RNN::SequentialCell') or not $self->_bidirectional),
"Bidirectional SequentialCell doesn't support zoneout. ".
"Please add ZoneoutCell to the cells underneath instead."
);
}
method reset()
{
$self->SUPER::reset;
$self->prev_output(undef);
}
method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states)
{
my ($cell, $p_outputs, $p_states) = ($self->base_cell, $self->zoneout_outputs, $self->zoneout_states);
my ($next_output, $next_states) = &{$cell}($inputs, $states);
my $mask = sub {
my ($p, $like) = @_;
AI::MXNet::Symbol->Dropout(
AI::MXNet::Symbol->ones_like(
$like
),
p => $p
);
};
my $prev_output = $self->prev_output // AI::MXNet::Symbol->zeros(shape => [0, 0]);
my $output = $p_outputs != 0
? AI::MXNet::Symbol->where(
&{$mask}($p_outputs, $next_output),
$next_output,
$prev_output
)
: $next_output;
my @states;
( run in 2.357 seconds using v1.01-cache-2.11-cpan-39bf76dae61 )