AI-MXNet
view release on metacpan or search on metacpan
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
=cut
=head1 DESCRIPTION
A very simple SGD optimizer with momentum and weight regularization.
Parameters
----------
learning_rate : float, optional
learning_rate of SGD
momentum : float, optional
momentum value
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
param_idx2name : hash of string/int to float, optional
special treat weight decay in parameter ends with bias, gamma, and beta
multi_precision: bool, optional
Flag to control the internal precision of the optimizer.
False results in using the same precision as the weights (default),
True makes internal 32-bit copy of the weights and applies gradients
in 32-bit precision even if actual weights used in the model have lower precision.
Turning this on can improve convergence and accuracy when training with float16.
=cut
package AI::MXNet::SGD;
use Mouse;
extends 'AI::MXNet::Optimizer';
has 'kwargs' => (is => "rw", isa => "HashRef[Num]");
has 'momentum' => (is => "rw", isa => "Num", default => 0);
has 'multi_precision' => (is => "ro", isa => "Bool", default => 0);
sub BUILD
{
my $self = shift;
$self->kwargs({ rescale_grad => $self->rescale_grad });
if($self->momentum)
{
$self->kwargs->{momentum} = $self->momentum;
}
if($self->clip_gradient)
{
$self->kwargs->{clip_gradient} = $self->clip_gradient;
}
}
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
my $momentum;
my $weight_master_copy;
if($self->multi_precision and $weight->dtype eq 'float16')
{
my $weight_master_copy = AI::MXNet::NDArray->array($weight, ctx => $weight->context, dtype => 'float32');
if($self->momentum != 0)
{
$momentum = AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype => 'float32');
}
return [$momentum, $weight_master_copy];
}
if($weight->dtype eq 'float16' and not $self->multi_precision)
{
AI::MXNet::Logging->warning(
"Accumulating with float16 in optimizer can lead to ".
"poor accuracy or slow convergence. ".
"Consider using multi_precision=True option of the ".
"SGD optimizer"
);
}
if($self->momentum != 0)
{
$momentum = AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype => $weight->dtype);
}
return $momentum;
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
Maybe[AI::MXNet::NDArray|ArrayRef[Maybe[AI::MXNet::NDArray]]] $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
my $kwargs = {
out => $weight,
lr => $lr,
wd => $wd,
%{ $self->kwargs }
};
my $use_multi_precision = ref($state) eq 'ARRAY';
if(not $use_multi_precision)
{
if(defined $state)
{
AI::MXNet::NDArray->sgd_mom_update(
$weight, $grad, $state, $kwargs
);
}
else
{
AI::MXNet::NDArray->sgd_update(
$weight, $grad, $kwargs
);
}
}
else
{
if(defined $state->[0])
{
AI::MXNet::NDArray->mp_sgd_mom_update(
$weight, $grad, $state->[0], $state->[1], $kwargs
);
}
else
{
AI::MXNet::NDArray->mp_sgd_update(
$weight, $grad, $state->[1], $kwargs
);
}
}
}
__PACKAGE__->register;
package AI::MXNet::DCASGD;
use Mouse;
use AI::MXNet::Base;
extends 'AI::MXNet::Optimizer';
=head1 NAME
AI::MXNet::DCASGD - DCASGD optimizer with momentum and weight regularization.
=cut
=head1 DESCRIPTION
DCASGD optimizer with momentum and weight regularization.
Implements paper "Asynchronous Stochastic Gradient Descent with
Delay Compensation for Distributed Deep Learning"
Parameters
----------
learning_rate : float, optional
learning_rate of SGD
momentum : float, optional
momentum value
lamda : float, optional
scale DC value
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
param_idx2name : hash ref of string/int to float, optional
special treat weight decay in parameter ends with bias, gamma, and beta
=cut
has 'momentum' => (is => 'ro', isa => 'Num', default => 0);
has 'lamda' => (is => 'ro', isa => 'Num', default => 0.04);
has 'weight_previous' => (is => 'rw', init_arg => undef);
sub BUILD
{
my $self = shift;
$self->weight_previous({});
}
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return [
$self->momentum ? AI::MXNet::NDArray->zeros(
$weight->shape, ctx => $weight->context, dtype => $weight->dtype
) : undef,
$weight->copy
];
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
Maybe[AI::MXNet::NDArray] $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
my ($mom, $weight_previous) = @{ $state };
if(defined $mom)
{
$mom *= $self->momentum;
$mom += -$lr * (
$grad + $wd * $weight
+
$self->lamda * $grad * $grad * ($weight - $weight_previous)
);
}
else
{
assert($self->momentum == 0);
$mom = -$lr * (
$grad + $wd * $weight
+
$self->lamda * $grad * $grad * ($weight - $weight_previous)
);
}
$weight_previous .= $weight;
$weight += $mom;
}
__PACKAGE__->register;
=head1 NAME
AI::MXNet::NAG - SGD with Nesterov weight handling.
=cut
=head1 DESCRIPTION
It is implemented according to
https://github.com/torch/optim/blob/master/sgd.lua
=cut
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
the code in this class was adapted from
https://github.com/mila-udem/blocks/blob/master/blocks/algorithms/__init__.py#L765
Parameters
----------
learning_rate : float, optional
Step size.
Default value is set to 0.001.
beta1 : float, optional
Exponential decay rate for the first moment estimates.
Default value is set to 0.9.
beta2 : float, optional
Exponential decay rate for the second moment estimates.
Default value is set to 0.999.
epsilon : float, optional
Default value is set to 1e-8.
decay_factor : float, optional
Default value is set to 1 - 1e-8.
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
=cut
package AI::MXNet::Adam;
use Mouse;
extends 'AI::MXNet::Optimizer';
has 'kwargs' => (is => "rw", isa => "HashRef[Num]");
has '+learning_rate' => (default => 0.001);
has 'beta1' => (is => "rw", isa => "Num", default => 0.9);
has 'beta2' => (is => "rw", isa => "Num", default => 0.999);
has 'epsilon' => (is => "rw", isa => "Num", default => 1e-8);
has 'decay_factor' => (is => "rw", isa => "Num", default => (1 - 1e-8));
sub BUILD
{
my $self = shift;
$self->kwargs({
rescale_grad => $self->rescale_grad,
beta1 => $self->beta1,
beta2 => $self->beta2,
epsilon => $self->epsilon
});
if($self->clip_gradient)
{
$self->kwargs->{clip_gradient} = $self->clip_gradient;
}
}
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return [AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
), # mean
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
) # variance
];
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
my $t = $self->_index_update_count->{$index};
my $coef1 = 1 - $self->beta1**$t;
my $coef2 = 1 - $self->beta2**$t;
$lr *= sqrt($coef2)/$coef1;
my ($mean, $var) = @{ $state };
AI::MXNet::NDArray->adam_update(
$weight, $grad, $mean, $var,
{
out => $weight,
lr => $lr,
wd => $wd,
%{ $self->kwargs }
}
);
}
__PACKAGE__->register;
=head1 NAME
AI::MXNet::AdaGrad - AdaGrad optimizer of Duchi et al., 2011
=cut
=head1 DESCRIPTION
AdaGrad optimizer of Duchi et al., 2011,
This code follows the version in http://arxiv.org/pdf/1212.5701v1.pdf Eq(5)
by Matthew D. Zeiler, 2012. AdaGrad will help the network to converge faster
in some cases.
Parameters
----------
learning_rate : float, optional
Step size.
Default value is set to 0.05.
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
eps: float, optional
A small float number to make the updating processing stable
Default value is set to 1e-7.
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
)
{
$self->_update_count($index);
my $wd = $self->_get_wd($index);
my $lr = $self->_get_lr($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
my ($dn, $n) = @{ $state };
$dn += $grad - (($n + $grad * $grad)->sqrt - $n->sqrt) * $weight / $lr;
$n += $grad * $grad;
$weight .= ($dn->sign * $self->lamda1 - $dn)
/
(($self->beta + $n->sqrt) / $lr + $wd) * ($dn->abs > $self->lamda1);
}
__PACKAGE__->register;
package AI::MXNet::Adamax;
=head1 NAME
AI::MXNet::Adamax
=cut
=head1 DESCRIPTION
It is a variant of Adam based on the infinity norm
available at http://arxiv.org/abs/1412.6980 Section 7.
This optimizer accepts the following parameters in addition to those accepted
AI::MXNet::Optimizer.
Parameters
----------
beta1 : float, optional
Exponential decay rate for the first moment estimates.
beta2 : float, optional
Exponential decay rate for the second moment estimates.
=cut
use Mouse;
extends 'AI::MXNet::Optimizer';
has '+learning_rate' => (default => 0.002);
has 'beta1' => (is => "ro", isa => "Num", default => 0.9);
has 'beta2' => (is => "ro", isa => "Num", default => 0.999);
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return [
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
), # mean
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
) # variance
];
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
my $wd = $self->_get_wd($index);
my $lr = $self->_get_lr($index);
$self->_update_count($index);
my $t = $self->_index_update_count->{$index};
$lr /= (1 - $self->beta1**$t);
$grad = $grad * $self->rescale_grad + $wd * $weight;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
# update m_t and u_t
my($m_t, $u_t) = @{ $state };
$m_t .= $self->beta1 * $m_t + (1 - $self->beta1) * $grad;
$u_t .= AI::MXNet::NDArray->maximum($self->beta2 * $u_t, $grad->abs);
# update weight
$weight -= $lr * $m_t / $u_t;
}
__PACKAGE__->register;
package AI::MXNet::Nadam;
=head1 NAME
AI::MXNet::Nadam
=cut
=head1 DESCRIPTION
The Nesterov Adam optimizer.
Much like Adam is essentially RMSprop with momentum,
Nadam is Adam RMSprop with Nesterov momentum available
at http://cs229.stanford.edu/proj2015/054_report.pdf.
This optimizer accepts the following parameters in addition to those accepted
AI::MXNet::Optimizer.
Parameters
----------
beta1 : float, optional
Exponential decay rate for the first moment estimates.
beta2 : float, optional
Exponential decay rate for the second moment estimates.
epsilon : float, optional
Small value to avoid division by 0.
schedule_decay : float, optional
Exponential decay rate for the momentum schedule
=cut
use Mouse;
extends 'AI::MXNet::Optimizer';
has '+learning_rate' => (default => 0.001);
has 'beta1' => (is => "ro", isa => "Num", default => 0.9);
has 'beta2' => (is => "ro", isa => "Num", default => 0.999);
has 'epsilon' => (is => "ro", isa => "Num", default => 1e-8);
has 'schedule_decay' => (is => "ro", isa => "Num", default => 0.004);
has 'm_schedule' => (is => "rw", default => 1, init_arg => undef);
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return [
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
), # mean
AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context,
dtype => $weight->dtype
) # variance
];
}
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
my $wd = $self->_get_wd($index);
my $lr = $self->_get_lr($index);
$self->_update_count($index);
my $t = $self->_index_update_count->{$index};
$grad = $grad * $self->rescale_grad + $wd * $weight;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
# warming momentum schedule
my $momentum_t = $self->beta1 * (1 - 0.5 * (0.96**($t * $self->schedule_decay)));
my $momentum_t_1 = $self->beta1 * (1 - 0.5 * (0.96**(($t + 1) * $self->schedule_decay)));
$self->m_schedule = $self->m_schedule * $momentum_t;
my $m_schedule_next = $self->m_schedule * $momentum_t_1;
# update m_t and v_t
my ($m_t, $v_t) = @{ $state };
$m_t .= $self->beta1 * $m_t + (1 - $self->beta1) * $grad;
$v_t .= $self->beta2 * $v_t + (1 - $self->beta2) * $grad * $grad;
my $grad_prime = $grad / (1 - $self->m_schedule);
my $m_t_prime = $m_t / (1 - $m_schedule_next);
my $v_t_prime = $v_t / (1 - $self->beta2**$t);
my $m_t_bar = (1 - $momentum_t) * $grad_prime + $momentum_t_1 * $m_t_prime;
# update weight
$weight -= $lr * $m_t_bar / (sqrt($v_t_prime) + $self->epsilon);
}
__PACKAGE__->register;
# updater for kvstore
package AI::MXNet::Updater;
use Mouse;
use Storable qw(thaw freeze);
use overload "&{}" => sub { my $self = shift; sub { $self->call(@_) } },
fallback => 1;
has "optimizer" => (is => "rw", isa => "AI::MXNet::Optimizer");
has "states" => (is => "rw", isa => "HashRef", default => sub { +{} });
has "states_synced" => (is => "rw", isa => "HashRef", default => sub { +{} });
method call(Index $index, AI::MXNet::NDArray $grad, AI::MXNet::NDArray $weight)
{
if(not exists $self->states->{ $index })
( run in 1.945 second using v1.01-cache-2.11-cpan-df04353d9ac )