AI-MXNet
view release on metacpan or search on metacpan
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
Create an optimizer with specified name.
Parameters
----------
name: str
Name of required optimizer. Should be the name
of a subclass of Optimizer. Case insensitive.
rescale_grad : float
Rescaling factor on gradient. Normally should be 1/batch_size.
kwargs: dict
Parameters for optimizer
Returns
-------
opt : Optimizer
The result optimizer.
=cut
method create_optimizer(Str $name, %kwargs)
{
if(exists $opt_registry{ lc $name })
{
my $rescale_grad = delete($kwargs{rescale_grad})//1;
return $opt_registry{ lc $name }->new(
rescale_grad => $rescale_grad,
%kwargs
);
}
confess("Cannot find optimizer $name");
}
*create = \&create_optimizer;
has 'rescale_grad' => (is => "rw", isa => "Num", default=>1);
has 'lr' => (is => "rw", isa => "Num");
has 'learning_rate' => (is => "rw", isa => "Num", default => 0.01);
has 'lr_scheduler' => (is => "rw", isa => "Maybe[AI::MXNet::LRScheduler]");
has 'wd' => (is => "rw", isa => "Num", default => 0);
has 'lr_mult' => (is => "rw", isa => "HashRef", default => sub { +{} });
has 'wd_mult' => (is => "rw", isa => "HashRef", , default => sub { +{} });
has 'num_update' => (is => "rw", isa => "Int");
has 'begin_num_update' => (is => "rw", isa => "Int", default => 0);
has '_index_update_count' => (is => "rw", isa => "HashRef", default => sub { +{} });
has 'clip_gradient' => (is => "rw", isa => "Maybe[Num]");
has 'param_idx2name' => (is => "rw", isa => "HashRef[Str]", default => sub { +{} });
has 'idx2name' => (is => "rw", isa => "HashRef[Str]");
has 'sym' => (is => "rw", isa => "Maybe[AI::MXNet::Symbol]");
sub BUILD
{
my $self = shift;
if($self->lr_scheduler)
{
$self->lr_scheduler->base_lr($self->learning_rate);
}
$self->lr($self->learning_rate);
$self->num_update($self->begin_num_update);
$self->idx2name({ %{ $self->param_idx2name } });
$self->set_lr_mult({});
$self->set_wd_mult({});
}
# Create additional optimizer state such as momentum.
# override in implementations.
method create_state($index, $weight){}
# Update the parameters. override in implementations
method update($index, $weight, $grad, $state){}
# set lr scale is deprecated. Use set_lr_mult instead.
method set_lr_scale($args_lrscale)
{
Carp::cluck("set lr scale is deprecated. Use set_lr_mult instead.");
}
=head2 set_lr_mult
Set individual learning rate multipler for parameters
Parameters
----------
args_lr_mult : dict of string/int to float
set the lr multipler for name/index to float.
setting multipler by index is supported for backward compatibility,
but we recommend using name and symbol.
=cut
method set_lr_mult(HashRef[Num] $args_lr_mult)
{
$self->lr_mult({});
if($self->sym)
{
my $attr = $self->sym->attr_dict();
for my $name (@{ $self->sym->list_arguments() })
{
if(exists $attr->{ $name } and exists $attr->{ $name }{ __lr_mult__ })
{
$self->lr_mult->{ $name } = $attr->{ $name }{ __lr_mult__ };
}
}
}
$self->lr_mult({ %{ $self->lr_mult }, %{ $args_lr_mult } });
}
=head2 set_wd_mult
Set individual weight decay multipler for parameters.
By default wd multipler is 0 for all params whose name doesn't
end with _weight, if param_idx2name is provided.
Parameters
----------
args_wd_mult : dict of string/int to float
set the wd multipler for name/index to float.
setting multipler by index is supported for backward compatibility,
but we recommend using name and symbol.
=cut
method set_wd_mult(HashRef[Num] $args_wd_mult)
{
$self->wd_mult({});
for my $n (values %{ $self->idx2name })
{
if(not $n =~ /(?:_weight|_gamma)$/)
{
$self->wd_mult->{ $n } = 0;
}
}
if($self->sym)
{
my $attr = $self->sym->attr_dict();
for my $name (@{ $self->sym->list_arguments() })
{
if(exists $attr->{ $name } and exists $attr->{ $name }{ __wd_mult__ })
{
$self->wd_mult->{ $name } = $attr->{ $name }{ __wd_mult__ };
}
}
}
$self->wd_mult({ %{ $self->wd_mult }, %{ $args_wd_mult } });
}
method _update_count(Index $index)
{
if(not exists $self->_index_update_count->{ $index })
{
$self->_index_update_count->{ $index } = $self->begin_num_update;
}
$self->_index_update_count->{ $index } += 1;
$self->num_update(max($self->_index_update_count->{ $index }, $self->num_update));
}
method _get_lr(Index $index)
{
my $lr;
if($self->lr_scheduler)
{
$lr = &{$self->lr_scheduler}($self->num_update);
}
else
{
$lr = $self->lr;
}
if(exists $self->lr_mult->{ $index })
{
$lr *= $self->lr_mult->{ $index };
}
elsif(exists $self->idx2name->{ $index })
{
$lr *= $self->lr_mult->{ $self->idx2name->{ $index } }//1;
}
return $lr;
}
method _get_wd(Index $index)
{
my $wd = $self->wd;
if(exists $self->wd_mult->{ $index })
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
package AI::MXNet::SLGD;
use Mouse;
extends 'AI::MXNet::Optimizer';
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return undef;
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
AI::MXNet::NDArray|Undef $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
$weight += - $lr/2 * ($grad + $wd * $weight)
+
AI::MXNet::Random->normal(
0, sqrt($lr),
$weight->shape,
$weight->context
);
}
__PACKAGE__->register;
=head1 NAME
AI::MXNet::Adam - Adam optimizer as described in [King2014]_.
=cut
=head1 DESCRIPTION
Adam optimizer as described in [King2014]_.
.. [King2014] Diederik Kingma, Jimmy Ba,
*Adam: A Method for Stochastic Optimization*,
http://arxiv.org/abs/1412.6980
the code in this class was adapted from
https://github.com/mila-udem/blocks/blob/master/blocks/algorithms/__init__.py#L765
Parameters
----------
learning_rate : float, optional
Step size.
Default value is set to 0.001.
beta1 : float, optional
Exponential decay rate for the first moment estimates.
Default value is set to 0.9.
beta2 : float, optional
Exponential decay rate for the second moment estimates.
Default value is set to 0.999.
epsilon : float, optional
Default value is set to 1e-8.
decay_factor : float, optional
Default value is set to 1 - 1e-8.
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
=cut
package AI::MXNet::Adam;
use Mouse;
extends 'AI::MXNet::Optimizer';
has 'kwargs' => (is => "rw", isa => "HashRef[Num]");
has '+learning_rate' => (default => 0.001);
has 'beta1' => (is => "rw", isa => "Num", default => 0.9);
has 'beta2' => (is => "rw", isa => "Num", default => 0.999);
has 'epsilon' => (is => "rw", isa => "Num", default => 1e-8);
has 'decay_factor' => (is => "rw", isa => "Num", default => (1 - 1e-8));
sub BUILD
{
my $self = shift;
$self->kwargs({
rescale_grad => $self->rescale_grad,
beta1 => $self->beta1,
beta2 => $self->beta2,
epsilon => $self->epsilon
});
if($self->clip_gradient)
{
$self->kwargs->{clip_gradient} = $self->clip_gradient;
}
}
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return [AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
), # mean
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
) # variance
];
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
my $t = $self->_index_update_count->{$index};
my $coef1 = 1 - $self->beta1**$t;
my $coef2 = 1 - $self->beta2**$t;
$lr *= sqrt($coef2)/$coef1;
my ($mean, $var) = @{ $state };
AI::MXNet::NDArray->adam_update(
$weight, $grad, $mean, $var,
{
out => $weight,
lr => $lr,
wd => $wd,
%{ $self->kwargs }
}
);
}
__PACKAGE__->register;
=head1 NAME
AI::MXNet::AdaGrad - AdaGrad optimizer of Duchi et al., 2011
=cut
=head1 DESCRIPTION
AdaGrad optimizer of Duchi et al., 2011,
This code follows the version in http://arxiv.org/pdf/1212.5701v1.pdf Eq(5)
by Matthew D. Zeiler, 2012. AdaGrad will help the network to converge faster
in some cases.
Parameters
----------
learning_rate : float, optional
Step size.
Default value is set to 0.05.
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
eps: float, optional
A small float number to make the updating processing stable
Default value is set to 1e-7.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
=cut
package AI::MXNet::AdaGrad;
use Mouse;
extends 'AI::MXNet::Optimizer';
has 'float_stable_eps' => (is => "rw", isa => "Num", default => 1e-7);
has '+learning_rate' => (default => 0.05);
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context
); # history
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
AI::MXNet::NDArray $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
my $history = $state;
$history += ($grad * $grad);
$weight += -$lr
*
(
$grad
/
AI::MXNet::NDArray->sqrt(
$history
+
$self->float_stable_eps
)
+
$wd * $weight
);
}
__PACKAGE__->register;
=head1 NAME
AI::MXNet::RMSProp - RMSProp optimizer of Tieleman & Hinton, 2012.
=cut
=head1 DESCRIPTION
RMSProp optimizer of Tieleman & Hinton, 2012,
For centered=False, the code follows the version in
http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf by
Tieleman & Hinton, 2012
For centered=True, the code follows the version in
http://arxiv.org/pdf/1308.0850v5.pdf Eq(38) - Eq(45) by Alex Graves, 2013.
Parameters
----------
learning_rate : float, optional
Step size.
Default value is set to 0.001.
gamma1: float, optional
decay factor of moving average for gradient^2.
Default value is set to 0.9.
gamma2: float, optional
"momentum" factor.
Default value if set to 0.9.
Only used if centered=True
epsilon : float, optional
Default value is set to 1e-8.
centered : bool, optional
Use Graves or Tielemans & Hintons version of RMSProp
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
clip_weights : float, optional
clip weights in range [-clip_weights, clip_weights]
=cut
package AI::MXNet::RMSProp;
use Mouse;
extends 'AI::MXNet::Optimizer';
has '+learning_rate' => (default => 0.001);
has 'gamma1' => (is => "ro", isa => "Num", default => 0.9);
has 'gamma2' => (is => "ro", isa => "Num", default => 0.9);
has 'epsilon' => (is => "ro", isa => "Num", default => 1e-8);
has 'centered' => (is => "ro", isa => "Bool", default => 0);
has 'clip_weights' => (is => "ro", isa => "Num");
has 'kwargs' => (is => "rw", init_arg => undef);
sub BUILD
{
my $self = shift;
$self->kwargs({
rescale_grad => $self->rescale_grad,
gamma1 => $self->gamma1,
epsilon => $self->epsilon
});
if($self->centered)
{
$self->kwargs->{gamma2} = $self->gamma2;
}
if($self->clip_gradient)
{
$self->kwargs->{clip_gradient} = $self->clip_gradient;
}
if($self->clip_weights)
{
$self->kwargs->{clip_weights} = $self->clip_weights;
}
}
# For centered=False: n
# For centered=True: n, g, delta
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return [
$self->centered
? (
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context
), # n
AI::MXNet::NDArray->zeros(
$weight->shape,
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
my $momentum_t_1 = $self->beta1 * (1 - 0.5 * (0.96**(($t + 1) * $self->schedule_decay)));
$self->m_schedule = $self->m_schedule * $momentum_t;
my $m_schedule_next = $self->m_schedule * $momentum_t_1;
# update m_t and v_t
my ($m_t, $v_t) = @{ $state };
$m_t .= $self->beta1 * $m_t + (1 - $self->beta1) * $grad;
$v_t .= $self->beta2 * $v_t + (1 - $self->beta2) * $grad * $grad;
my $grad_prime = $grad / (1 - $self->m_schedule);
my $m_t_prime = $m_t / (1 - $m_schedule_next);
my $v_t_prime = $v_t / (1 - $self->beta2**$t);
my $m_t_bar = (1 - $momentum_t) * $grad_prime + $momentum_t_1 * $m_t_prime;
# update weight
$weight -= $lr * $m_t_bar / (sqrt($v_t_prime) + $self->epsilon);
}
__PACKAGE__->register;
# updater for kvstore
package AI::MXNet::Updater;
use Mouse;
use Storable qw(thaw freeze);
use overload "&{}" => sub { my $self = shift; sub { $self->call(@_) } },
fallback => 1;
has "optimizer" => (is => "rw", isa => "AI::MXNet::Optimizer");
has "states" => (is => "rw", isa => "HashRef", default => sub { +{} });
has "states_synced" => (is => "rw", isa => "HashRef", default => sub { +{} });
method call(Index $index, AI::MXNet::NDArray $grad, AI::MXNet::NDArray $weight)
{
if(not exists $self->states->{ $index })
{
$self->states->{ $index } = $self->optimizer->create_state($index, $weight);
$self->states_synced->{ $index } = 1;
}
elsif(not $self->states_synced->{ $index })
{
$self->states->{ $index } = $self->sync_state_context($self->states->{ $index }, $weight->context);
$self->states_synced->{ $index } = 1;
}
$self->optimizer->update($index, $weight, $grad, $self->states->{ $index });
}
*slice = *call;
method sync_state_context(Maybe[AI::MXNet::NDArray|ArrayRef[AI::MXNet::NDArray]] $state, AI::MXNet::Context $context)
{
if(blessed $state)
{
return $state->as_in_context($context);
}
elsif(ref $state)
{
return [map { $self->sync_state_context($_, $context) } @{ $state }];
}
return $state;
}
method set_states($states)
{
my $thawed_states = thaw($states);
$self->states($thawed_states);
%{ $self->states_synced } = map { $_ => 0 } keys %{ $thawed_states };
}
method get_states()
{
return freeze($self->states);
}
package AI::MXNet::Optimizer;
method get_updater(AI::MXNet::Optimizer $optimizer)
{
return AI::MXNet::Updater->new(optimizer => $optimizer);
}
1;
( run in 2.285 seconds using v1.01-cache-2.11-cpan-483215c6ad5 )