AI-MXNet
view release on metacpan or search on metacpan
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
{
if(defined $state->[0])
{
AI::MXNet::NDArray->mp_sgd_mom_update(
$weight, $grad, $state->[0], $state->[1], $kwargs
);
}
else
{
AI::MXNet::NDArray->mp_sgd_update(
$weight, $grad, $state->[1], $kwargs
);
}
}
}
__PACKAGE__->register;
package AI::MXNet::DCASGD;
use Mouse;
use AI::MXNet::Base;
extends 'AI::MXNet::Optimizer';
=head1 NAME
AI::MXNet::DCASGD - DCASGD optimizer with momentum and weight regularization.
=cut
=head1 DESCRIPTION
DCASGD optimizer with momentum and weight regularization.
Implements paper "Asynchronous Stochastic Gradient Descent with
Delay Compensation for Distributed Deep Learning"
Parameters
----------
learning_rate : float, optional
learning_rate of SGD
momentum : float, optional
momentum value
lamda : float, optional
scale DC value
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
param_idx2name : hash ref of string/int to float, optional
special treat weight decay in parameter ends with bias, gamma, and beta
=cut
has 'momentum' => (is => 'ro', isa => 'Num', default => 0);
has 'lamda' => (is => 'ro', isa => 'Num', default => 0.04);
has 'weight_previous' => (is => 'rw', init_arg => undef);
sub BUILD
{
my $self = shift;
$self->weight_previous({});
}
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return [
$self->momentum ? AI::MXNet::NDArray->zeros(
$weight->shape, ctx => $weight->context, dtype => $weight->dtype
) : undef,
$weight->copy
];
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
Maybe[AI::MXNet::NDArray] $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
my ($mom, $weight_previous) = @{ $state };
if(defined $mom)
{
$mom *= $self->momentum;
$mom += -$lr * (
$grad + $wd * $weight
+
$self->lamda * $grad * $grad * ($weight - $weight_previous)
);
}
else
{
assert($self->momentum == 0);
$mom = -$lr * (
$grad + $wd * $weight
+
$self->lamda * $grad * $grad * ($weight - $weight_previous)
);
}
$weight_previous .= $weight;
$weight += $mom;
}
__PACKAGE__->register;
=head1 NAME
AI::MXNet::NAG - SGD with Nesterov weight handling.
=cut
=head1 DESCRIPTION
It is implemented according to
https://github.com/torch/optim/blob/master/sgd.lua
=cut
package AI::MXNet::NAG;
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
if($state)
{
my $mom = $state;
$mom *= $self->momentum;
$grad += $wd * $weight;
$mom += $grad;
$grad += $self->momentum * $mom;
$weight += -$lr * $grad;
}
else
{
confess("momentum != 0") unless $self->momentum == 0;
$weight += -$lr * ($grad + $wd * $weight);
}
}
__PACKAGE__->register;
=head1 NAME
AI::MXNet::SLGD - Stochastic Langevin Dynamics Updater to sample from a distribution.
=cut
=head1 DESCRIPTION
Stochastic Langevin Dynamics Updater to sample from a distribution.
Parameters
----------
learning_rate : float, optional
learning_rate of SGD
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
param_idx2name : dict of string/int to float, optional
special treat weight decay in parameter ends with bias, gamma, and beta
=cut
package AI::MXNet::SLGD;
use Mouse;
extends 'AI::MXNet::Optimizer';
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return undef;
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
AI::MXNet::NDArray|Undef $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
$weight += - $lr/2 * ($grad + $wd * $weight)
+
AI::MXNet::Random->normal(
0, sqrt($lr),
$weight->shape,
$weight->context
);
}
__PACKAGE__->register;
=head1 NAME
AI::MXNet::Adam - Adam optimizer as described in [King2014]_.
=cut
=head1 DESCRIPTION
Adam optimizer as described in [King2014]_.
.. [King2014] Diederik Kingma, Jimmy Ba,
*Adam: A Method for Stochastic Optimization*,
http://arxiv.org/abs/1412.6980
the code in this class was adapted from
https://github.com/mila-udem/blocks/blob/master/blocks/algorithms/__init__.py#L765
Parameters
----------
learning_rate : float, optional
Step size.
Default value is set to 0.001.
beta1 : float, optional
Exponential decay rate for the first moment estimates.
Default value is set to 0.9.
beta2 : float, optional
Exponential decay rate for the second moment estimates.
Default value is set to 0.999.
epsilon : float, optional
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
+
$wd * $weight
);
}
__PACKAGE__->register;
=head1 NAME
AI::MXNet::RMSProp - RMSProp optimizer of Tieleman & Hinton, 2012.
=cut
=head1 DESCRIPTION
RMSProp optimizer of Tieleman & Hinton, 2012,
For centered=False, the code follows the version in
http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf by
Tieleman & Hinton, 2012
For centered=True, the code follows the version in
http://arxiv.org/pdf/1308.0850v5.pdf Eq(38) - Eq(45) by Alex Graves, 2013.
Parameters
----------
learning_rate : float, optional
Step size.
Default value is set to 0.001.
gamma1: float, optional
decay factor of moving average for gradient^2.
Default value is set to 0.9.
gamma2: float, optional
"momentum" factor.
Default value if set to 0.9.
Only used if centered=True
epsilon : float, optional
Default value is set to 1e-8.
centered : bool, optional
Use Graves or Tielemans & Hintons version of RMSProp
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
clip_weights : float, optional
clip weights in range [-clip_weights, clip_weights]
=cut
package AI::MXNet::RMSProp;
use Mouse;
extends 'AI::MXNet::Optimizer';
has '+learning_rate' => (default => 0.001);
has 'gamma1' => (is => "ro", isa => "Num", default => 0.9);
has 'gamma2' => (is => "ro", isa => "Num", default => 0.9);
has 'epsilon' => (is => "ro", isa => "Num", default => 1e-8);
has 'centered' => (is => "ro", isa => "Bool", default => 0);
has 'clip_weights' => (is => "ro", isa => "Num");
has 'kwargs' => (is => "rw", init_arg => undef);
sub BUILD
{
my $self = shift;
$self->kwargs({
rescale_grad => $self->rescale_grad,
gamma1 => $self->gamma1,
epsilon => $self->epsilon
});
if($self->centered)
{
$self->kwargs->{gamma2} = $self->gamma2;
}
if($self->clip_gradient)
{
$self->kwargs->{clip_gradient} = $self->clip_gradient;
}
if($self->clip_weights)
{
$self->kwargs->{clip_weights} = $self->clip_weights;
}
}
# For centered=False: n
# For centered=True: n, g, delta
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return [
$self->centered
? (
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context
), # n
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context
), # g
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context
)
) # delta
: (
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context
), # n
)
];
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
my $lr = $self->_get_lr($index);
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
$lr /= (1 - $self->beta1**$t);
$grad = $grad * $self->rescale_grad + $wd * $weight;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
# update m_t and u_t
my($m_t, $u_t) = @{ $state };
$m_t .= $self->beta1 * $m_t + (1 - $self->beta1) * $grad;
$u_t .= AI::MXNet::NDArray->maximum($self->beta2 * $u_t, $grad->abs);
# update weight
$weight -= $lr * $m_t / $u_t;
}
__PACKAGE__->register;
package AI::MXNet::Nadam;
=head1 NAME
AI::MXNet::Nadam
=cut
=head1 DESCRIPTION
The Nesterov Adam optimizer.
Much like Adam is essentially RMSprop with momentum,
Nadam is Adam RMSprop with Nesterov momentum available
at http://cs229.stanford.edu/proj2015/054_report.pdf.
This optimizer accepts the following parameters in addition to those accepted
AI::MXNet::Optimizer.
Parameters
----------
beta1 : float, optional
Exponential decay rate for the first moment estimates.
beta2 : float, optional
Exponential decay rate for the second moment estimates.
epsilon : float, optional
Small value to avoid division by 0.
schedule_decay : float, optional
Exponential decay rate for the momentum schedule
=cut
use Mouse;
extends 'AI::MXNet::Optimizer';
has '+learning_rate' => (default => 0.001);
has 'beta1' => (is => "ro", isa => "Num", default => 0.9);
has 'beta2' => (is => "ro", isa => "Num", default => 0.999);
has 'epsilon' => (is => "ro", isa => "Num", default => 1e-8);
has 'schedule_decay' => (is => "ro", isa => "Num", default => 0.004);
has 'm_schedule' => (is => "rw", default => 1, init_arg => undef);
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return [
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
), # mean
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
) # variance
];
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
my $wd = $self->_get_wd($index);
my $lr = $self->_get_lr($index);
$self->_update_count($index);
my $t = $self->_index_update_count->{$index};
$grad = $grad * $self->rescale_grad + $wd * $weight;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
# warming momentum schedule
my $momentum_t = $self->beta1 * (1 - 0.5 * (0.96**($t * $self->schedule_decay)));
my $momentum_t_1 = $self->beta1 * (1 - 0.5 * (0.96**(($t + 1) * $self->schedule_decay)));
$self->m_schedule = $self->m_schedule * $momentum_t;
my $m_schedule_next = $self->m_schedule * $momentum_t_1;
# update m_t and v_t
my ($m_t, $v_t) = @{ $state };
$m_t .= $self->beta1 * $m_t + (1 - $self->beta1) * $grad;
$v_t .= $self->beta2 * $v_t + (1 - $self->beta2) * $grad * $grad;
my $grad_prime = $grad / (1 - $self->m_schedule);
my $m_t_prime = $m_t / (1 - $m_schedule_next);
my $v_t_prime = $v_t / (1 - $self->beta2**$t);
my $m_t_bar = (1 - $momentum_t) * $grad_prime + $momentum_t_1 * $m_t_prime;
# update weight
$weight -= $lr * $m_t_bar / (sqrt($v_t_prime) + $self->epsilon);
}
__PACKAGE__->register;
# updater for kvstore
( run in 0.527 second using v1.01-cache-2.11-cpan-d80b1682f3f )