view release on metacpan or  search on metacpan

lib/AI/MXNet/Initializer.pm  view on Meta::CPAN

package AI::MXNet::InitDesc;
use Mouse;
use AI::MXNet::Function::Parameters;

=head1 NAME

    AI::MXNet::InitDesc - A container for the initialization pattern serialization.

=head2 new

    Parameters
    ---------
    name : str
        name of variable
    attrs : hash ref of str to str
        attributes of this variable taken from AI::MXNet::Symbol->attr_dict
=cut
has 'name'        => (is => 'ro', isa => 'Str', required => 1);
has 'attrs'       => (is => 'rw', isa => 'HashRef[Str]', lazy => 1, default => sub { +{} });
use overload '""' => sub { shift->name };
around BUILDARGS => sub {
    my $orig  = shift;
    my $class = shift;
    return $class->$orig(name => $_[0]) if @_ == 1;
    return $class->$orig(@_);
};

# Base class for Initializers
package AI::MXNet::Initializer;
use Mouse;
use AI::MXNet::Base qw(:DEFAULT pzeros pceil);
use AI::MXNet::NDArray;
use JSON::PP;
use overload "&{}" => sub { my $self = shift; sub { $self->call(@_) } },
             '""'  => sub {
                my $self = shift;
                my ($name) = ref($self) =~ /::(\w+)$/;
                encode_json(
                    [lc $name,
                        $self->kwargs//{ map { $_ => "".$self->$_ } $self->meta->get_attribute_list }
                ]);
             },
             fallback => 1;
has 'kwargs' => (is => 'rw', init_arg => undef, isa => 'HashRef');
has '_verbose'    => (is => 'rw', isa => 'Bool', lazy => 1, default => 0);
has '_print_func' => (is => 'rw', isa => 'CodeRef', lazy => 1,
    default => sub {
        return sub {
            my $x = shift;
            return ($x->norm/sqrt($x->size))->asscalar;
        };
    }
);

=head1 NAME

    AI::MXNet::Initializer - Base class for all Initializers

=head2 register

    Register an initializer class to the AI::MXNet::Initializer factory.
=cut

=head2 set_verbosity

    Switch on/off verbose mode

    Parameters
    ----------
    $verbose : bool
        switch on/off verbose mode
    $print_func : CodeRef
        A function that computes statistics of initialized arrays.
        Takes an AI::MXNet::NDArray and returns a scalar. Defaults to mean
        absolute value |x|/size(x)
=cut

method set_verbosity(Bool $verbose=0, CodeRef $print_func=)
{
    $self->_verbose($verbose);
    $self->_print_func($print_func) if defined $print_func;
}

method _verbose_print($desc, $init, $arr)
{
    if($self->_verbose and defined $self->_print_func)
    {
        AI::MXNet::Logging->info('Initialized %s as %s: %s', $desc, $init, $self->_print_func->($arr));
    }
}

my %init_registry;
method get_init_registry()
{
    return \%init_registry;
}

method register()
{
    my ($name) = $self =~ /::(\w+)$/;
    my $orig_name = $name;
    $name         = lc $name;
    if(exists $init_registry{ $name })
    {
        my $existing = $init_registry{ $name };
        warn(
            "WARNING: New initializer $self.$name" 
            ."is overriding existing initializer $existing.$name"
        );
    }
    $init_registry{ $name } = $self;
    {
        no strict 'refs';
        no warnings 'redefine';
        *{"$orig_name"} = sub { shift; $self->new(@_) };
        *InitDesc       = sub { shift; AI::MXNet::InitDesc->new(@_) };
    }
}

=head2 init

    Parameters
    ----------
    $desc : AI::MXNet::InitDesc|str
        a name of corresponding ndarray
        or the object that describes the initializer.

    $arr : AI::MXNet::NDArray
        an ndarray to be initialized.
=cut
method call(Str|AI::MXNet::InitDesc $desc, AI::MXNet::NDArray $arr)
{
    return $self->_legacy_init($desc, $arr) unless blessed $desc;
    my $init = $desc->attrs->{ __init__ };
    if($init)
    {
      my ($klass, $kwargs) = @{ decode_json($init) };
      $self->get_init_registry->{ lc $klass }->new(%{ $kwargs })->_init_weight("$desc", $arr);
      $self->_verbose_print($desc, $init, $arr);
    }
    else
    {
        $desc = "$desc";
        if($desc =~ /(weight|bias|gamma|beta)$/)
        {
            my $method = "_init_$1";
            $self->$method($desc, $arr);
            $self->_verbose_print($desc, $1, $arr);
        }
        else
        {
            $self->_init_default($desc, $arr)
        }
    }
}


method _legacy_init(Str $name, AI::MXNet::NDArray $arr)
{
    warnings::warnif(
        'deprecated',
        'Calling initializer with init($str, $NDArray) has been deprecated.'.
        'please use init(mx->init->InitDesc(...), NDArray) instead.'
    );
    if($name =~ /^upsampling/)
    {
        $self->_init_bilinear($name, $arr);
    }
    elsif($name =~ /^stn_loc/ and $name =~ /weight$/)
    {
        $self->_init_zero($name, $arr);
    }
    elsif($name =~ /^stn_loc/ and $name =~ /bias$/)
    {
        $self->_init_loc_bias($name, $arr);
    }
    elsif($name =~ /bias$/)
    {
        $self->_init_bias($name, $arr);
    }
    elsif($name =~ /gamma$/)
    {
        $self->_init_gamma($name, $arr);
    }
    elsif($name =~ /beta$/)
    {
        $self->_init_beta($name, $arr);
    }
    elsif($name =~ /weight$/)
    {
        $self->_init_weight($name, $arr);
    }
    elsif($name =~ /moving_mean$/)
    {
        $self->_init_zero($name, $arr);
    }
    elsif($name =~ /moving_var$/)
    {
        $self->_init_one($name, $arr);
    }
    elsif($name =~ /moving_inv_var$/)
    {
        $self->_init_zero($name, $arr);
    }
    elsif($name =~ /moving_avg$/)
    {
        $self->_init_zero($name, $arr);
    }

lib/AI/MXNet/Initializer.pm  view on Meta::CPAN

method _init_weight(Str $name, AI::MXNet::NDArray $arr)
{
    AI::MXNet::Random->uniform(-$self->scale, $self->scale, { out => $arr });
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::Normal - Initialize the weight with gaussian random values.
=cut

=head1 DESCRIPTION

    Initialize the weight with gaussian random values contained within of [0, sigma]

    Parameters
    ----------
    sigma : float, optional
        Standard deviation for the gaussian distribution.
=cut

package AI::MXNet::Normal;
use Mouse;
extends 'AI::MXNet::Initializer';
has "sigma" => (is => "ro", isa => "Num", default => 0.01);
around BUILDARGS => sub {
    my $orig  = shift;
    my $class = shift;
    return $class->$orig(sigma => $_[0]) if @_ == 1;
    return $class->$orig(@_);
};

method _init_weight(Str $name, AI::MXNet::NDArray $arr)
{
    AI::MXNet::Random->normal(0, $self->sigma, { out => $arr });
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::Orthogonal - Intialize the weight as an Orthogonal matrix.
=cut

=head1 DESCRIPTION

    Intialize weight as Orthogonal matrix

    Parameters
    ----------
    scale : float, optional
        scaling factor of weight

    rand_type: string optional
        use "uniform" or "normal" random number to initialize weight

    Reference
    ---------
    Exact solutions to the nonlinear dynamics of learning in deep linear neural networks
    arXiv preprint arXiv:1312.6120 (2013).
=cut

package AI::MXNet::Orthogonal;
use AI::MXNet::Base;
use Mouse;
use AI::MXNet::Types;
extends 'AI::MXNet::Initializer';
has "scale" => (is => "ro", isa => "Num", default => 1.414);
has "rand_type" => (is => "ro", isa => enum([qw/uniform normal/]), default => 'uniform');

method _init_weight(Str $name, AI::MXNet::NDArray $arr)
{
    my @shape = @{ $arr->shape };
    my $nout = $shape[0];
    my $nin = AI::MXNet::NDArray->size([@shape[1..$#shape]]);
    my $tmp = AI::MXNet::NDArray->zeros([$nout, $nin]);
    if($self->rand_type eq 'uniform')
    {
        AI::MXNet::Random->uniform(-1, 1, { out => $tmp });
    }
    else
    {
        AI::MXNet::Random->normal(0, 1, { out => $tmp });
    }
    $tmp = $tmp->aspdl;
    my ($u, $s, $v) = svd($tmp);
    my $q;
    if(join(',', @{ $u->shape->unpdl }) eq join(',', @{ $tmp->shape->unpdl }))
    {
        $q = $u;
    }
    else
    {
        $q = $v;
    }
    $q = $self->scale * $q->reshape(reverse(@shape));
    $arr .= $q;
}

*slice = *call;
__PACKAGE__->register;

=head1 NAME

    AI::MXNet::Xavier - Initialize the weight with Xavier or similar initialization scheme.
=cut

=head1 DESCRIPTION

    Parameters
    ----------
    rnd_type: str, optional
        Use gaussian or uniform.
    factor_type: str, optional
        Use avg, in, or out.
    magnitude: float, optional
        The scale of the random number range.
=cut

package AI::MXNet::Xavier;