view release on metacpan or  search on metacpan

lib/AI/MXNet/Initializer.pm  view on Meta::CPAN

        A function that computes statistics of initialized arrays.
        Takes an AI::MXNet::NDArray and returns a scalar. Defaults to mean
        absolute value |x|/size(x)
=cut

method set_verbosity(Bool $verbose=0, CodeRef $print_func=)
{
    $self->_verbose($verbose);
    $self->_print_func($print_func) if defined $print_func;
}

method _verbose_print($desc, $init, $arr)
{
    if($self->_verbose and defined $self->_print_func)
    {
        AI::MXNet::Logging->info('Initialized %s as %s: %s', $desc, $init, $self->_print_func->($arr));
    }
}

my %init_registry;
method get_init_registry()
{
    return \%init_registry;
}

method register()
{
    my ($name) = $self =~ /::(\w+)$/;
    my $orig_name = $name;
    $name         = lc $name;
    if(exists $init_registry{ $name })
    {
        my $existing = $init_registry{ $name };
        warn(
            "WARNING: New initializer $self.$name" 
            ."is overriding existing initializer $existing.$name"
        );
    }
    $init_registry{ $name } = $self;
    {
        no strict 'refs';
        no warnings 'redefine';
        *{"$orig_name"} = sub { shift; $self->new(@_) };
        *InitDesc       = sub { shift; AI::MXNet::InitDesc->new(@_) };
    }
}

=head2 init

    Parameters
    ----------
    $desc : AI::MXNet::InitDesc|str
        a name of corresponding ndarray
        or the object that describes the initializer.

    $arr : AI::MXNet::NDArray
        an ndarray to be initialized.
=cut
method call(Str|AI::MXNet::InitDesc $desc, AI::MXNet::NDArray $arr)
{
    return $self->_legacy_init($desc, $arr) unless blessed $desc;
    my $init = $desc->attrs->{ __init__ };
    if($init)
    {
      my ($klass, $kwargs) = @{ decode_json($init) };
      $self->get_init_registry->{ lc $klass }->new(%{ $kwargs })->_init_weight("$desc", $arr);
      $self->_verbose_print($desc, $init, $arr);
    }
    else
    {
        $desc = "$desc";
        if($desc =~ /(weight|bias|gamma|beta)$/)
        {
            my $method = "_init_$1";
            $self->$method($desc, $arr);
            $self->_verbose_print($desc, $1, $arr);
        }
        else
        {
            $self->_init_default($desc, $arr)
        }
    }
}


method _legacy_init(Str $name, AI::MXNet::NDArray $arr)
{
    warnings::warnif(
        'deprecated',
        'Calling initializer with init($str, $NDArray) has been deprecated.'.
        'please use init(mx->init->InitDesc(...), NDArray) instead.'
    );
    if($name =~ /^upsampling/)
    {
        $self->_init_bilinear($name, $arr);
    }
    elsif($name =~ /^stn_loc/ and $name =~ /weight$/)
    {
        $self->_init_zero($name, $arr);
    }
    elsif($name =~ /^stn_loc/ and $name =~ /bias$/)
    {
        $self->_init_loc_bias($name, $arr);
    }
    elsif($name =~ /bias$/)
    {
        $self->_init_bias($name, $arr);
    }
    elsif($name =~ /gamma$/)
    {
        $self->_init_gamma($name, $arr);
    }
    elsif($name =~ /beta$/)
    {
        $self->_init_beta($name, $arr);
    }
    elsif($name =~ /weight$/)
    {
        $self->_init_weight($name, $arr);
    }
    elsif($name =~ /moving_mean$/)

lib/AI/MXNet/Initializer.pm  view on Meta::CPAN

    Parameters
    ----------
    forget_bias: float,a bias for the forget gate.
    Jozefowicz et al. 2015 recommends setting this to 1.0.
=cut

use Mouse;
extends 'AI::MXNet::Initializer';
has 'forget_bias' => (is => 'ro', isa => 'Num', required => 1);

method _init_weight(Str $name, AI::MXNet::NDArray $arr)
{
    $arr .= 0;
    # in the case of LSTMCell the forget gate is the second
    # gate of the 4 LSTM gates, we modify the according values.
    my $num_hidden = int($arr->shape->[0] / 4);
    $arr->slice([$num_hidden, 2*$num_hidden-1]) .= $self->forget_bias;
}

__PACKAGE__->register;

package AI::MXNet::FusedRNN;
use Mouse;
use JSON::PP;
extends 'AI::MXNet::Initializer';

=head1 NAME

    AI::MXNet::FusedRNN - Custom initializer for fused RNN cells.
=cut

=head1 DESCRIPTION

    Initializes parameters for fused rnn layer.

    Parameters
    ----------
    init : Initializer
        initializer applied to unpacked weights.
    All parameters below must be exactly the same as ones passed to the
    FusedRNNCell constructor.

    num_hidden : int
    num_layers : int
    mode : str
    bidirectional : bool
    forget_bias : float
=cut

has 'init'          => (is => 'rw', isa => 'Str|AI::MXNet::Initializer', required => 1);
has 'forget_bias'   => (is => 'ro', isa => 'Num', default => 1);
has [qw/num_hidden
       num_layers/] => (is => 'ro', isa => 'Int', required => 1);
has 'mode'          => (is => 'ro', isa => 'Str', required => 1);
has 'bidirectional' => (is => 'ro', isa => 'Bool', default => 0);

sub BUILD
{
    my $self = shift;
    if(not blessed $self->init)
    {
        my ($klass, $kwargs);
        eval {
            ($klass, $kwargs) = @{ decode_json($self->init) };
        };
        confess("FusedRNN failed to init $@") if $@;
        $self->init($self->get_init_registry->{ lc $klass }->new(%$kwargs));
    }
}

method _init_weight($name, $arr)
{
    my $cell = AI::MXNet::RNN::FusedCell->new(
        num_hidden    => $self->num_hidden,
        num_layers    => $self->num_layers,
        mode          => $self->mode,
        bidirectional => $self->bidirectional,
        forget_bias   => $self->forget_bias,
        prefix        => ''
    );

    my $args = $cell->unpack_weights({ parameters => $arr });
    for my $name (keys %{ $args })
    {
        my $desc = AI::MXNet::InitDesc->new(name => $name);
        # for lstm bias, we use a custom initializer
        # which adds a bias to the forget gate
        if($self->mode eq 'lstm' and $name =~ /f_bias$/)
        {
            $args->{$name} .= $self->forget_bias;
        }
        else
        {
            &{$self->init}($desc, $args->{$name});
        }
    }

    $arr .= $cell->pack_weights($args)->{parameters};
}

__PACKAGE__->register;

1;