AI-MXNet
view release on metacpan or search on metacpan
Revision history for Perl extension AI::MXNet
1.0102 Sun Aug 6 16:55:08 PDT 2017
- bugfixes in Image.pm, updated tests, added PearsonCorrelation metric, added Convolutional RNN modules.
1.0101 Sun Jul 2 17:16:01 PDT 2017
- reworked CachedOp, two new optimizers, auto module reshape, using strings to index the kvstore.
1.01 Sat Jun 10 23:57:27 PDT 2017
- sync with python.
0.9507 Thu May 11 17:04:44 PDT 2017
- added AutoGrad, bugfixes.
0.9506 Sat Apr 29 20:26:50 PDT 2017
- Ftrl optimizer, new tests, bugfixes.
- LR Scheduler bugfix.
0.9503 Wed Apr 19 13:33:57 PDT 2017
- added an example of generation of inferred text via pre-trained RNN.
- bugfixes/tests.
0.9502 Sat Apr 15 17:18:21 PDT 2017
- optimizations/bugfixes.
0.9501 Sat Apr 8 13:01:00 PDT 2017
- ZoneoutCell, nd inferred reshape and moveaxis, cosmetic changes to Image iter,
pod reworked to be readable via metacpan.
0.95 Sun Mar 26 17:42:02 PDT 2017
- docs, bugfixes, tests in order to be visible on http://mxnet.io
0.03 Tue Feb 14 07:28:11 PST 2017
- sync up with current state of the Python inteface.
- high level RNN support.
0.02 Tue Feb 14 07:28:11 PST 2017
},
"name" : "AI-MXNet",
"no_index" : {
"directory" : [
"t",
"inc"
]
},
"prereqs" : {
"build" : {
"requires" : {}
},
"configure" : {
"requires" : {
"ExtUtils::MakeMaker" : "6.30"
}
},
"runtime" : {
"requires" : {
"AI::MXNetCAPI" : "1.0102",
"AI::NNVMCAPI" : "1.01",
"Function::Parameters" : "1.0705",
"GraphViz" : "2.14",
"Mouse" : "v2.1.0",
"PDL" : "2.007"
}
},
"test" : {
"requires" : {}
}
},
"release_status" : "stable",
"version" : "1.0102"
}
---
abstract: 'Perl interface to MXNet machine learning library'
author:
- 'Sergey Kolychev <sergeykolychev.github@gmail.com>'
build_requires: {}
configure_requires:
ExtUtils::MakeMaker: '6.30'
dynamic_config: 0
generated_by: 'ExtUtils::MakeMaker version 7.24, CPAN::Meta::Converter version 2.143240'
license: apache
meta-spec:
url: http://module-build.sourceforge.net/META-spec-v1.4.html
version: '1.4'
name: AI-MXNet
no_index:
directory:
- t
- inc
requires:
AI::MXNetCAPI: '1.0102'
AI::NNVMCAPI: '1.01'
Function::Parameters: '1.0705'
GraphViz: '2.14'
Mouse: v2.1.0
PDL: '2.007'
version: '1.0102'
examples/calculator.pl view on Meta::CPAN
## to train our network
sub samples {
my($batch_size, $func) = @_;
# get samples
my $n = 16384;
## creates a pdl with $n rows and two columns with random
## floats in the range between 0 and 1
my $data = PDL->random(2, $n);
## creates the pdl with $n rows and one column with labels
## labels are floats that either sum or product, etc of
## two random values in each corresponding row of the data pdl
my $label = $func->($data->slice('0,:'), $data->slice('1,:'));
# partition into train/eval sets
my $edge = int($n / 8);
my $validation_data = $data->slice(":,0:@{[ $edge - 1 ]}");
my $validation_label = $label->slice(":,0:@{[ $edge - 1 ]}");
my $train_data = $data->slice(":,$edge:");
my $train_label = $label->slice(":,$edge:");
# build iterators around the sets
return(mx->io->NDArrayIter(
batch_size => $batch_size,
examples/calculator.pl view on Meta::CPAN
my $data = mx->sym->Variable('data');
my $ln = mx->sym->exp(mx->sym->FullyConnected(
data => mx->sym->log($data),
num_hidden => 1,
));
my $wide = mx->sym->Concat($data, $ln);
my $fc = mx->sym->FullyConnected(
$wide,
num_hidden => 1
);
return mx->sym->MAERegressionOutput(data => $fc, name => 'softmax');
}
sub learn_function {
my(%args) = @_;
my $func = $args{func};
my $batch_size = $args{batch_size}//128;
my($train_iter, $eval_iter) = samples($batch_size, $func);
my $sym = nn_fc();
## call as ./calculator.pl 1 to just print model and exit
examples/calculator.pl view on Meta::CPAN
my $model = mx->mod->Module(
symbol => $sym,
context => mx->cpu(),
);
$model->fit($train_iter,
eval_data => $eval_iter,
optimizer => 'adam',
optimizer_params => {
learning_rate => $args{lr}//0.01,
rescale_grad => 1/$batch_size,
lr_scheduler => AI::MXNet::FactorScheduler->new(
step => 100,
factor => 0.99
)
},
eval_metric => 'mse',
num_epoch => $args{epoch}//25,
);
# refit the model for calling on 1 sample at a time
my $iter = mx->io->NDArrayIter(
batch_size => 1,
data => PDL->pdl([[ 0, 0 ]]),
label => PDL->pdl([[ 0 ]]),
);
$model->reshape(
data_shapes => $iter->provide_data,
label_shapes => $iter->provide_label,
);
# wrap a helper around making predictions
my ($arg_params) = $model->get_params;
for my $k (sort keys %$arg_params)
{
print "$k -> ". $arg_params->{$k}->aspdl."\n";
}
examples/char_lstm.pl view on Meta::CPAN
--gpus list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu.
Increase batch size when using multiple gpus for best performance.
--kv-store key-value store type, default='device'
--num-epochs max num of epochs, default=25
--lr initial learning rate, default=0.01
--optimizer the optimizer type, default='adam'
--mom momentum for sgd, default=0.0
--wd weight decay for sgd, default=0.00001
--batch-size the batch size type, default=32
--bidirectional use bidirectional cell, default false (0)
--disp-batches show progress for every n batches, default=50
--chkp-prefix prefix for checkpoint files, default='lstm_'
--cell-mode RNN cell mode (LSTM, GRU, RNN, default=LSTM)
--sample-size a size of inferred sample text (default=10000) after each epoch
--chkp-epoch save checkpoint after this many epoch, default=1 (saving every checkpoint)
=cut
package AI::MXNet::RNN::IO::ASCIIIterator;
use Mouse;
extends AI::MXNet::DataIter;
examples/char_lstm.pl view on Meta::CPAN
dtype => $self->dtype
)
]);
$self->provide_label([
AI::MXNet::DataDesc->new(
name => $self->label_name,
shape => $shape,
dtype => $self->dtype
)
]);
$self->reset;
}
method reset()
{
$self->counter(0);
@{ $self->idx } = List::Util::shuffle(@{ $self->idx });
}
method next()
{
return undef if $self->counter == @{$self->idx};
my $offset = $self->idx->[$self->counter]*$self->batch_size*$self->seq_size + $self->seq_counter;
my $data = $self->nd->slice(
[$offset, $offset + $self->batch_size*$self->seq_size-1]
)->reshape([$self->batch_size, $self->seq_size]);
my $label = $self->nd->slice(
[$offset + 1 , $offset + $self->batch_size*$self->seq_size]
)->reshape([$self->batch_size, $self->seq_size]);
$self->seq_counter($self->seq_counter + 1);
if($self->seq_counter == $seq_size - 1)
{
$self->counter($self->counter + 1);
$self->seq_counter(0);
}
return AI::MXNet::DataBatch->new(
data => [$data],
label => [$label],
provide_data => [
examples/char_lstm.pl view on Meta::CPAN
}
$stack->add($cell);
}
my $data = mx->sym->Variable('data');
my $label = mx->sym->Variable('softmax_label');
my $embed = mx->sym->Embedding(
data => $data, input_dim => scalar(keys %vocabulary),
output_dim => $num_embed, name => 'embed'
);
$stack->reset;
my ($outputs, $states) = $stack->unroll($seq_size, inputs => $embed, merge_outputs => 1);
my $pred = mx->sym->Reshape($outputs, shape => [-1, $num_hidden*(1+($bidirectional ? 1 : 0))]);
$pred = mx->sym->FullyConnected(data => $pred, num_hidden => $data_iter->vocab_size, name => 'pred');
$label = mx->sym->Reshape($label, shape => [-1]);
my $net = mx->sym->SoftmaxOutput(data => $pred, label => $label, name => 'softmax');
my $contexts;
if(defined $gpus)
{
$contexts = [map { mx->gpu($_) } split(/,/, $gpus)];
examples/char_lstm.pl view on Meta::CPAN
$model->fit(
$data_iter,
eval_metric => mx->metric->Perplexity,
kvstore => $kv_store,
optimizer => $optimizer,
optimizer_params => {
learning_rate => $lr,
momentum => $mom,
wd => $wd,
clip_gradient => 5,
rescale_grad => 1/$batch_size,
lr_scheduler => AI::MXNet::FactorScheduler->new(step => 1000, factor => 0.99)
},
initializer => mx->init->Xavier(factor_type => "in", magnitude => 2.34),
num_epoch => $num_epoch,
batch_end_callback => mx->callback->Speedometer($batch_size, $disp_batches),
($chkp_epoch ? (epoch_end_callback => [mx->rnn->do_rnn_checkpoint($stack, $chkp_prefix, $chkp_epoch), \&sample]) : ())
);
sub sample {
return if not $sample_size;
$model->reshape(data_shapes=>[['data',[1, $seq_size]]], label_shapes=>[['softmax_label',[1, $seq_size]]]);
my $input = mx->nd->array($fdata->slice([0, $seq_size-1]))->reshape([1, $seq_size]);
$| = 1;
for (0..$sample_size-1)
{
$model->forward(mx->io->DataBatch(data=>[$input]), is_train => 0);
my $prob = $model->get_outputs(0)->[0][0]->at($seq_size-1)->aspdl;
my $next_char = Math::Random::Discrete->new($prob->reshape(-1)->unpdl, [0..scalar(keys %vocabulary)-1])->rand;
print "$reverse_vocab{$next_char}";
$input->at(0)->slice([0, $seq_size-2]) .= $input->at(0)->slice([1, $seq_size-1])->copy;
$input->at(0)->at($seq_size-1) .= $next_char;
}
$model->reshape(data_shapes=>[['data',[$batch_size, $seq_size]]], label_shapes=>[['softmax_label',[$batch_size, $seq_size]]]);
}
examples/cudnn_lstm_bucketing.pl view on Meta::CPAN
--num-seq sequence size, default=32
--gpus list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu.
Increase batch size when using multiple gpus for best performance.
--kv-store key-value store type, default='device'
--num-epochs max num of epochs, default=25
--lr initial learning rate, default=0.01
--optimizer the optimizer type, default='adam'
--mom momentum for sgd, default=0.0
--wd weight decay for sgd, default=0.00001
--batch-size the batch size type, default=32
--disp-batches show progress for every n batches, default=50
--model-prefix prefix for checkpoint files for loading/saving, default='lstm_'
--load-epoch load from epoch
--stack-rnn stack rnn to reduce communication overhead (1,0 default 0)
--bidirectional whether to use bidirectional layers (1,0 default 0)
--dropout dropout probability (1.0 - keep probability), default 0
=cut
$bidirectional = $bidirectional ? 1 : 0;
$stack_rnn = $stack_rnn ? 1 : 0;
examples/cudnn_lstm_bucketing.pl view on Meta::CPAN
)->unfuse()
}
my $sym_gen = sub {
my $seq_len = shift;
my $data = mx->sym->Variable('data');
my $label = mx->sym->Variable('softmax_label');
my $embed = mx->sym->Embedding(
data => $data, input_dim => scalar(keys %$vocab),
output_dim => $num_embed, name => 'embed'
);
$stack->reset;
my ($outputs, $states) = $stack->unroll($seq_len, inputs => $embed, merge_outputs => 1);
my $pred = mx->sym->Reshape($outputs, shape => [-1, $num_hidden*(1+$bidirectional)]);
$pred = mx->sym->FullyConnected(data => $pred, num_hidden => scalar(keys %$vocab), name => 'pred');
$label = mx->sym->Reshape($label, shape => [-1]);
$pred = mx->sym->SoftmaxOutput(data => $pred, label => $label, name => 'softmax');
return ($pred, ['data'], ['softmax_label']);
};
my $contexts;
if($gpus)
{
examples/lstm_bucketing.pl view on Meta::CPAN
--num-embed embedding layer size, default=200
--gpus list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu.
Increase batch size when using multiple gpus for best performance.
--kv-store key-value store type, default='device'
--num-epochs max num of epochs, default=25
--lr initial learning rate, default=0.01
--optimizer the optimizer type, default='sgd'
--mom momentum for sgd, default=0.0
--wd weight decay for sgd, default=0.00001
--batch-size the batch size type, default=32
--disp-batches show progress for every n batches, default=50
--chkp-prefix prefix for checkpoint files, default='lstm_'
--chkp-epoch save checkpoint after this many epoch, default=0 (saving checkpoints is disabled)
=cut
func tokenize_text($fname, :$vocab=, :$invalid_label=-1, :$start_label=0)
{
open(F, $fname) or die "Can't open $fname: $!";
my @lines = map { my $l = [split(/ /)]; shift(@$l); $l } (<F>);
my $sentences;
($sentences, $vocab) = mx->rnn->encode_sentences(
examples/lstm_bucketing.pl view on Meta::CPAN
}
my $sym_gen = sub {
my $seq_len = shift;
my $data = mx->sym->Variable('data');
my $label = mx->sym->Variable('softmax_label');
my $embed = mx->sym->Embedding(
data => $data, input_dim => scalar(keys %$vocabulary),
output_dim => $num_embed, name => 'embed'
);
$stack->reset;
my ($outputs, $states) = $stack->unroll($seq_len, inputs => $embed, merge_outputs => 1);
my $pred = mx->sym->Reshape($outputs, shape => [-1, $num_hidden]);
$pred = mx->sym->FullyConnected(data => $pred, num_hidden => scalar(keys %$vocabulary), name => 'pred');
$label = mx->sym->Reshape($label, shape => [-1]);
$pred = mx->sym->SoftmaxOutput(data => $pred, label => $label, name => 'softmax');
return ($pred, ['data'], ['softmax_label']);
};
my $contexts;
if(defined $gpus)
examples/mnist.pl view on Meta::CPAN
$win->signal_connect(delete_event => sub { Gtk2->main_quit() });
$win->add($sw);
$win->show_all();
Gtk2->main();
}
#show_sample();
sub to4d {
my($img) = @_;
return $img->reshape(28, 28, 1, ($img->dims)[2])->float / 255;
}
my $batch_size = 100;
my $train_iter = mx->io->NDArrayIter(
data => to4d($train_img),
label => $train_lbl,
batch_size => $batch_size,
shuffle => 1,
);
my $val_iter = mx->io->NDArrayIter(
examples/mnist.pl view on Meta::CPAN
#my $shape = { data => [ $batch_size, 1, 28, 28 ] };
#show_network(mx->viz->plot_network($mlp, shape => $shape));
my $model = mx->mod->Module(
symbol => $mlp, # network structure
);
$model->fit(
$train_iter, # training data
num_epoch => 10, # number of data passes for training
eval_data => $val_iter, # validation data
batch_end_callback => mx->callback->Speedometer($batch_size, 200), # output progress for each 200 data batches
optimizer => 'adam',
);
lib/AI/MXNet.pm view on Meta::CPAN
batch_size=>$batch_size, shuffle=>1, flat=>0, silent=>0});
my $n_epoch = 1;
my $mod = mx->mod->new(symbol => $softmax);
$mod->fit(
$train_dataiter,
eval_data => $val_dataiter,
optimizer_params=>{learning_rate=>0.01, momentum=> 0.9},
num_epoch=>$n_epoch
);
my $res = $mod->score($val_dataiter, mx->metric->create('acc'));
ok($res->{accuracy} > 0.8);
=head1 DESCRIPTION
Perl interface to MXNet machine learning library.
=head1 BUGS AND INCOMPATIBILITIES
Parity with Python inteface is mostly achieved, few deprecated
and not often used features left unported for now.
=head1 SEE ALSO
http://mxnet.io/
https://github.com/dmlc/mxnet/tree/master/perl-package
Function::Parameters, Mouse
=head1 AUTHOR
Sergey Kolychev, <sergeykolychev.github@gmail.com>
lib/AI/MXNet/Base.pm view on Meta::CPAN
return $lo;
}
=head2 pdl_shuffle
Shuffle the pdl by the last dimension
Parameters
-----------
PDL $pdl
$preshuffle Maybe[ArrayRef[Index]], if defined the array elements are used
as shuffled last dimension's indexes
=cut
sub pdl_shuffle
{
my ($pdl, $preshuffle) = @_;
my $c = $pdl->copy;
my @shuffle = $preshuffle ? @{ $preshuffle } : shuffle(0..$pdl->dim(-1)-1);
my $rem = $pdl->ndims-1;
for my $i (0..$pdl->dim(-1)-1)
{
$c->slice(('X')x$rem, $i) .= $pdl->slice(('X')x$rem, $shuffle[$i])
}
$c;
}
=head2 assert
lib/AI/MXNet/Callback.pm view on Meta::CPAN
my ($iter_no, $sym, $arg, $aux) = @_;
if(($iter_no + 1) % $period == 0)
{
$mod->save_checkpoint($prefix, $iter_no + 1, $save_optimizer_states);
}
}
}
=head2 log_train_metric
Callback to log the training evaluation result every period.
Parameters
----------
$period : Int
The number of batches after which to log the training evaluation metric.
$auto_reset : Bool
Whether to reset the metric after the logging.
Returns
-------
$callback : sub ref
The callback function that can be passed as iter_epoch_callback to fit.
=cut
method log_train_metric(Int $period, Int $auto_reset=0)
{
return sub {
my ($param) = @_;
if($param->nbatch % $period == 0 and defined $param->eval_metric)
{
my $name_value = $param->eval_metric->get_name_value;
while(my ($name, $value) = each %{ $name_value })
{
AI::MXNet::Logging->info(
"Iter[%d] Batch[%d] Train-%s=%f",
$param->epoch, $param->nbatch, $name, $value
);
}
$param->eval_metric->reset if $auto_reset;
}
}
}
package AI::MXNet::Speedometer;
use Mouse;
use Time::HiRes qw/time/;
extends 'AI::MXNet::Callback';
=head1 NAME
lib/AI/MXNet/Callback.pm view on Meta::CPAN
Calculate and log training speed periodically.
Parameters
----------
batch_size: int
batch_size of data
frequent: int
How many batches between calculations.
Defaults to calculating & logging every 50 batches.
auto_reset: Bool
Reset the metric after each log, defaults to true.
=cut
has 'batch_size' => (is => 'ro', isa => 'Int', required => 1);
has 'frequent' => (is => 'ro', isa => 'Int', default => 50);
has 'init' => (is => 'rw', isa => 'Int', default => 0);
has 'tic' => (is => 'rw', isa => 'Num', default => 0);
has 'last_count' => (is => 'rw', isa => 'Int', default => 0);
has 'auto_reset' => (is => 'ro', isa => 'Bool', default => 1);
method call(AI::MXNet::BatchEndParam $param)
{
my $count = $param->nbatch;
if($self->last_count > $count)
{
$self->init(0);
}
$self->last_count($count);
if($self->init)
{
if(($count % $self->frequent) == 0)
{
my $speed = $self->frequent * $self->batch_size / (time - $self->tic);
if(defined $param->eval_metric)
{
my $name_value = $param->eval_metric->get_name_value;
$param->eval_metric->reset if $self->auto_reset;
while(my ($name, $value) = each %{ $name_value })
{
AI::MXNet::Logging->info(
"Epoch[%d] Batch [%d]\tSpeed: %.2f samples/sec\tTrain-%s=%f",
$param->epoch, $count, $speed, $name, $value
);
}
}
else
{
lib/AI/MXNet/Callback.pm view on Meta::CPAN
}
else
{
$self->init(1);
$self->tic(time);
}
}
*slice = \&call;
package AI::MXNet::ProgressBar;
use Mouse;
extends 'AI::MXNet::Callback';
=head1 NAME
AI::MXNet::ProgressBar - A callback to show a progress bar.
=head1 DESCRIPTION
Shows a progress bar.
Parameters
----------
total: Int
batch size, default is 1
length: Int
the length of the progress bar, default is 80 chars
=cut
has 'length' => (is => 'ro', isa => 'Int', default => 80);
has 'total' => (is => 'ro', isa => 'Int', required => 1);
method call(AI::MXNet::BatchEndParam $param)
{
my $count = $param->nbatch;
my $filled_len = int(0.5 + $self->length * $count / $self->total);
my $percents = int(100.0 * $count / $self->total) + 1;
lib/AI/MXNet/Callback.pm view on Meta::CPAN
);
}
}
package AI::MXNet::Callback;
method Speedometer(@args)
{
AI::MXNet::Speedometer->new(
@args == 3 ?
(batch_size => $args[0], frequent => $args[1], auto_reset => $args[2])
: @args == 2 ?
(batch_size => $args[0], frequent => $args[1])
: (batch_size => $args[0])
)
}
method ProgressBar(@args)
{
AI::MXNet::ProgressBar->new(
@args == 2 ? (total => $args[0], 'length' => $args[1]) : (total => $args[0])
)
}
method LogValidationMetricsCallback()
{
AI::MXNet::LogValidationMetricsCallback->new
}
1;
lib/AI/MXNet/Context.pm view on Meta::CPAN
This class governs the device context of AI::MXNet::NDArray objects.
=cut
=head2
Constructing a context.
Parameters
----------
device_type : {'cpu', 'gpu'} or Context.
String representing the device type
device_id : int (default=0)
The device id of the device, needed for GPU
=cut
=head2 cpu
Returns a CPU context.
Parameters
----------
device_id : int, optional
The device id of the device. device_id is not needed for CPU.
This is included to make interface compatible with GPU.
Returns
-------
context : AI::MXNet::Context
The corresponding CPU context.
=cut
method cpu(Int $device_id=0)
{
return $self->new(device_type => 'cpu', device_id => $device_id);
}
=head2 gpu
Returns a GPU context.
Parameters
----------
device_id : int, optional
Returns
-------
context : AI::MXNet::Context
The corresponding GPU context.
=cut
method gpu(Int $device_id=0)
{
return $self->new(device_type => 'gpu', device_id => $device_id);
}
=head2 current_context
Returns the current context.
lib/AI/MXNet/Executor.pm view on Meta::CPAN
check_call(
AI::MXNetCAPI::ExecutorSetMonitorCallback(
$self->handle,
$self->_monitor_callback
)
);
}
=head2 arg_dict
Get a hash ref representation of the argument arrays.
Returns
-------
arg_dict : HashRef[AI::MXNet::NDArray]
The map that maps a name of the arguments to the NDArrays.
=cut
method arg_dict()
{
if(not defined $self->_arg_dict)
lib/AI/MXNet/Executor.pm view on Meta::CPAN
$self->_symbol->list_arguments(),
$self->arg_arrays
)
);
}
return $self->_arg_dict;
}
=head2 grad_dict
Get a hash ref representation of the gradient arrays.
Returns
-------
grad_dict : HashRef[AI::MXNet::NDArray]
The map that maps a name of the arguments to the gradient NDArrays.
=cut
method grad_dict()
{
if(not defined $self->_grad_dict)
lib/AI/MXNet/Executor.pm view on Meta::CPAN
$self->_symbol->list_arguments(),
$self->grad_arrays
)
);
}
return $self->_grad_dict;
}
=head2 aux_dict
Get a hash ref representation of the auxiliary states arrays.
Returns
-------
aux_dict : HashRef[AI::MXNet::NDArray]
The map that maps a name of the auxiliary states to the NDArrays.
=cut
method aux_dict()
{
if(not defined $self->_aux_dict)
lib/AI/MXNet/Executor.pm view on Meta::CPAN
$self->_symbol->list_auxiliary_states(),
$self->aux_arrays()
)
);
}
return $self->_aux_dict;
}
=head2 output_dict
Get a hash ref representation of the output arrays.
Returns
-------
output_dict : HashRef[AI::MXNet::NDArray]
The map that maps a name of the outputs to the NDArrays.
=cut
method output_dict()
{
if(not defined $self->_output_dict)
lib/AI/MXNet/Executor.pm view on Meta::CPAN
$array->astype($dst->dtype)->copyto($dst);
}
elsif(not $allow_extra_params)
{
confess("Found name \"$name\" that is not in the arguments");
}
}
}
}
=head2 reshape
Returns new executor with the same symbol and shared memory,
but different input/output shapes.
For runtime reshaping, variable length sequences, etc.
The returned executor shares state with the current one,
and cannot be used in parallel with it.
Parameters
----------
$kwargs : HashRef[Shape]
new shape for arguments.
:$partial_shaping : bool
Whether to allow changing the shape of unspecified arguments.
:$allow_up_sizing : bool
Whether to allow allocating new ndarrays that's larger than the original.
Returns
-------
$exec : AI::MXNet::Executor
A new executor that shares memory with self.
=cut
method reshape(HashRef[Shape] $kwargs, Int :$partial_shaping=0, Int :$allow_up_sizing=0)
{
my ($arg_shapes, undef, $aux_shapes) = $self->_symbol->infer_shape(%{ $kwargs });
confess("Insufficient argument shapes provided.")
unless defined $arg_shapes;
my %new_arg_dict;
my %new_grad_dict;
my $i = 0;
for my $name (@{ $self->_symbol->list_arguments() })
{
my $new_shape = $arg_shapes->[$i];
lib/AI/MXNet/Executor.pm view on Meta::CPAN
{
$new_grad_dict{ $name } = AI::MXNet::NDArray->empty(
$new_shape,
ctx => $darr->context,
dtype => $arr->dtype
);
}
}
else
{
$new_arg_dict{ $name } = $arr->reshape($new_shape);
if(defined $darr)
{
$new_grad_dict{ $name } = $darr->reshape($new_shape);
}
}
}
else
{
confess(
"Shape of unspecified array arg:$name changed. "
."This can cause the new executor to not share parameters "
."with the old one. Please check for error in network."
."If this is intended, set partial_shaping=True to suppress this warning."
);
}
$i++;
}
my %new_aux_dict;
$i = 0;
for my $name (@{ $self->_symbol->list_auxiliary_states() })
{
my $new_shape = $aux_shapes->[$i];
my $arr = $self->aux_arrays->[$i];
lib/AI/MXNet/Executor.pm view on Meta::CPAN
."to enable allocation of new arrays."
) unless $allow_up_sizing;
$new_aux_dict{ $name } = AI::MXNet::NDArray->empty(
$new_shape,
ctx => $arr->context,
dtype => $arr->dtype
);
}
else
{
$new_aux_dict{ $name } = $arr->reshape($new_shape);
}
}
else
{
confess(
"Shape of unspecified array aux:$name changed. "
."This can cause the new executor to not share parameters "
."with the old one. Please check for error in network."
."If this is intended, set partial_shaping=True to suppress this warning."
);
}
$i++;
}
return $self->_symbol->bind(
ctx => $self->_ctx,
args => \%new_arg_dict,
args_grad => \%new_grad_dict,
grad_req => $self->_grad_req,
aux_states => \%new_aux_dict,
lib/AI/MXNet/Executor/Group.pm view on Meta::CPAN
}
else
{
my $ctx = $tensors->[0]->context;
push @rets, AI::MXNet::NDArray->concat((map { $_->as_in_context($ctx) } @$tensors), { dim => $axis });
}
}
else
{
# negative axis means the there is no batch_size axis, and all the
# results should be the same on each device. We simply take the
# first one, without checking they are actually the same
push @rets, $tensors->[0];
}
}, $outputs, $major_axis);
return \@rets;
}
## TODO
## this class is here because of https://github.com/gfx/p5-Mouse/pull/67
## once 2.4.7 version of Mouse in Ubuntu for affected Perl version
lib/AI/MXNet/Executor/Group.pm view on Meta::CPAN
A array ref of strings, indicating the names of parameters (e.g. weights, filters, etc.)
in the computation graph.
for_training : Bool
Indicate whether the executors should be bind for training. When not doing training,
the memory for gradients will not be allocated.
inputs_need_grad : Bool
Indicate whether the gradients for the input data should be computed. This is currently
not used. It will be useful for implementing composition of modules.
shared_group : AI::MXNet::DataParallelExecutorGroup
Default is undef. This is used in bucketing. When not undef, it should be a executor
group corresponding to a different bucket. In other words, it will correspond to a different
symbol with the same set of parameters (e.g. unrolled RNNs with different lengths).
In this case the memory regions of the parameters will be shared.
logger : Logger
Default is AI::MXNet::Logging->get_logger.
fixed_param_names: Maybe[ArrayRef[Str]]
Indicate parameters to be fixed during training. Parameters in this array ref will not allocate
space for gradient, nor do gradient calculation.
grad_req : ArrayRef[GradReq]|HashRef[GradReq]|GradReq
Requirement for gradient accumulation. Can be 'write', 'add', or 'null'
(default to 'write').
lib/AI/MXNet/Executor/Group.pm view on Meta::CPAN
$self->_p->batch_size($batch_size);
$self->_p->slices(AI::MXNet::Executor::Group::_split_input_slice($self->_p->batch_size, $self->workload));
}
}, $data_shapes, $major_axis);
return $major_axis;
}
# Collect internal arrays from executors.
method _collect_arrays()
{
# convenient data structures
$self->_p->data_arrays([]);
for my $d (@{ $self->data_shapes })
{
my $name = $d->name;
my @tmp;
for my $i (0..@{ $self->_p->execs }-1)
{
push @tmp, [ $self->_p->slices->[$i], $self->_p->execs->[$i]->arg_dict->{$name} ];
}
push @{ $self->_p->data_arrays }, \@tmp;
lib/AI/MXNet/Executor/Group.pm view on Meta::CPAN
for my $exec (@{ $self->_p->execs })
{
push @tmp, $exec->aux_arrays->[$i];
}
push @{ $self->_p->aux_arrays }, \@tmp;
}
}
=head2 bind_exec
Bind executors on their respective devices.
Parameters
----------
$data_shapes : ArrayRef[AI::MXNet::DataDesc]
$label_shapes : Maybe[ArrayRef[AI::MXNet::DataDesc]]
$shared_group : Maybe[AI::MXNet::DataParallelExecutorGroup]
$reshape : Bool
=cut
method bind_exec(
ArrayRef[AI::MXNet::DataDesc] $data_shapes,
Maybe[ArrayRef[AI::MXNet::DataDesc]] $label_shapes=,
Maybe[AI::MXNet::DataParallelExecutorGroup] $shared_group=,
Bool $reshape=0
)
{
assert($reshape or not @{ $self->_p->execs });
$self->_p->batch_size(undef);
# calculate workload and bind executors
$self->_p->data_layouts($self->decide_slices($data_shapes));
# call it to make sure labels has the same batch size as data
if(defined $label_shapes)
{
$self->_p->label_layouts($self->decide_slices($label_shapes));
}
for my $i (0..@{ $self->contexts }-1)
{
my $data_shapes_i = $self->_sliced_shape($data_shapes, $i, $self->_p->data_layouts);
my $label_shapes_i = [];
if(defined $label_shapes)
{
$label_shapes_i = $self->_sliced_shape($label_shapes, $i, $self->_p->label_layouts);
}
if($reshape)
{
my %combined_hash = map { $_->name => $_->shape } (@{ $data_shapes_i }, @{ $label_shapes_i });
$self->_p->execs->[$i] = $self->_p->_default_execs->[$i]->reshape(
\%combined_hash,
allow_up_sizing => 1,
);
}
else
{
push @{ $self->_p->execs }, $self->_bind_ith_exec($i, $data_shapes_i, $label_shapes_i, $shared_group);
}
}
$self->data_shapes($data_shapes);
$self->label_shapes($label_shapes);
$self->_collect_arrays;
}
=head2 reshape
Reshape executors.
Parameters
----------
$data_shapes : ArrayRef[AI::MXNet::DataDesc]
$label_shapes : Maybe[ArrayRef[AI::MXNet::DataDesc]]
=cut
method reshape(
ArrayRef[AI::MXNet::DataDesc] $data_shapes,
Maybe[ArrayRef[AI::MXNet::DataDesc]] $label_shapes=
)
{
return if($data_shapes eq $self->data_shapes and $label_shapes eq $self->label_shapes);
if (not defined $self->_p->_default_execs)
{
$self->_p->_default_execs([@{ $self->_p->execs }]);
}
$self->bind_exec($data_shapes, $label_shapes, undef, 1);
lib/AI/MXNet/Executor/Group.pm view on Meta::CPAN
=head2 get_outputs
Gets outputs of the previous forward computation.
Parameters
----------
merge_multi_context : bool
Default is 1. In the case when data-parallelism is used, the outputs
will be collected from multiple devices. A 1 value indicates that we
should merge the collected results so that they look like from a single
executor.
Returns
-------
If merge_multi_context is 1, it is [$out1, $out2]. Otherwise, it
is [[$out1_dev1, $out1_dev2], [$out2_dev1, $out2_dev2]]. All the output
elements are `AI::MXNet::NDArray`.
=cut
method get_outputs(Bool $merge_multi_context=1)
lib/AI/MXNet/Executor/Group.pm view on Meta::CPAN
}
if($merge_multi_context)
{
$outputs = AI::MXNet::Executor::Group::_merge_multi_context($outputs, $self->_p->output_layouts);
}
return $outputs;
}
=head2 get_input_grads
Get the gradients with respect to the inputs of the module.
Parameters
----------
merge_multi_context : bool
Default is 1. In the case when data-parallelism is used, the outputs
will be collected from multiple devices. A 1 value indicates that we
should merge the collected results so that they look like from a single
executor.
Returns
-------
If merge_multi_context is 1, it is [$grad1, $grad2]. Otherwise, it
is [[$grad1_dev1, $grad1_dev2], [$grad2_dev1, $grad2_dev2]]. All the output
elements are AI::MXNet::NDArray.
=cut
method get_input_grads(Bool $merge_multi_context=1)
lib/AI/MXNet/IO.pm view on Meta::CPAN
[$self->name, $self->shape];
}
=head1 NAME
AI::MXNet::DataDesc - A container class for describing the data layout.
=cut
=head2 get_batch_axis
Get the dimension that corresponds to the batch size.
Parameters
----------
layout : str
layout string. For example, "NCHW".
Returns
-------
An axis indicating the batch_size dimension. When data-parallelism is
used, the data will be automatically split and concatenate along the batch_size
lib/AI/MXNet/IO.pm view on Meta::CPAN
use overload '<>' => sub { shift->next },
'@{}' => sub { shift->list };
=head1 NAME
AI::MXNet::DataIter - A parent class for MXNet data iterators.
=cut
has 'batch_size' => (is => 'rw', isa => 'Int', default => 0);
=head2 reset
Reset the iterator.
=cut
method reset(){}
=head2 list
Returns remaining iterator items as an array ref.
=cut
method list()
{
my @ret;
while(<$self>)
lib/AI/MXNet/IO.pm view on Meta::CPAN
=head1 DESCRIPTION
Resize a DataIter to a given number of batches per epoch.
May produce incomplete batch in the middle of an epoch due
to the padding from internal iterator.
Parameters
----------
data_iter : DataIter
Internal data iterator.
size : number of batches per epoch to resize to.
reset_internal : whether to reset internal iterator on ResizeIter.reset
=cut
has 'data_iter' => (is => 'ro', isa => 'AI::MXnet::DataIter', required => 1);
has 'size' => (is => 'ro', isa => 'Int', required => 1);
has 'reset_internal' => (is => 'rw', isa => 'Int', default => 1);
has 'cur' => (is => 'rw', isa => 'Int', default => 0);
has 'current_batch' => (is => 'rw', isa => 'Maybe[AI::MXNet::DataBatch]');
has [qw/provide_data
default_bucket_key
provide_label
batch_size/] => (is => 'rw', init_arg => undef);
sub BUILD
{
my $self = shift;
$self->provide_data($self->data_iter->provide_data);
$self->provide_label($self->data_iter->provide_label);
$self->batch_size($self->data_iter->batch_size);
if($self->data_iter->can('default_bucket_key'))
{
$self->default_bucket_key($self->data_iter->default_bucket_key);
}
}
method reset()
{
$self->cur(0);
if($self->reset_internal)
{
$self->data_iter->reset;
}
}
method iter_next()
{
return 0 if($self->cur == $self->size);
$self->current_batch($self->data_iter->next);
if(not defined $self->current_batch)
{
$self->data_iter->reset;
$self->current_batch($self->data_iter->next);
}
$self->cur($self->cur + 1);
return 1;
}
method get_data()
{
return $self->current_batch->data;
}
lib/AI/MXNet/IO.pm view on Meta::CPAN
{
return [map {
my ($k, $v) = @{ $_ };
my $shape = $v->shape;
$shape->[0] = $self->batch_size;
AI::MXNet::DataDesc->new(name => $k, shape => $shape, dtype => $v->dtype)
} @{ $self->label }];
}
# Ignore roll over data and set to start
method hard_reset()
{
$self->cursor(-$self->batch_size);
}
method reset()
{
if($self->last_batch_handle eq 'roll_over' and $self->cursor > $self->num_data)
{
$self->cursor(-$self->batch_size + ($self->cursor%$self->num_data)%$self->batch_size);
}
else
{
$self->cursor(-$self->batch_size);
}
}
lib/AI/MXNet/IO.pm view on Meta::CPAN
}
else
{
return undef;
}
}
# Load data from underlying arrays, internal use only
method _getdata($data_source)
{
confess("DataIter needs reset.") unless $self->cursor < $self->num_data;
if(($self->cursor + $self->batch_size) <= $self->num_data)
{
return [
map {
$_->[1]->slice([$self->cursor,$self->cursor+$self->batch_size-1])
} @{ $data_source }
];
}
else
{
lib/AI/MXNet/IO.pm view on Meta::CPAN
This can be used to test the speed of network without taking
the loading delay into account.
=cut
method debug_skip_load()
{
$self->_debug_skip_load(1);
AI::MXNet::Logging->info('Set debug_skip_load to be true, will simply return first batch');
}
method reset()
{
$self->_debug_at_begin(1);
$self->first_batch(undef);
check_call(AI::MXNetCAPI::DataIterBeforeFirst($self->handle));
}
method next()
{
if($self->_debug_skip_load and not $self->_debug_at_begin)
{
lib/AI/MXNet/IO.pm view on Meta::CPAN
index => $self->getindex
);
}
if(defined $self->first_batch)
{
my $batch = $self->first_batch;
$self->first_batch(undef);
return $batch
}
$self->_debug_at_begin(0);
my $next_res = check_call(AI::MXNetCAPI::DataIterNext($self->handle));
if($next_res)
{
return AI::MXNet::DataBatch->new(
data => [$self->getdata],
label => [$self->getlabel],
pad => $self->getpad,
index => $self->getindex
);
}
else
{
lib/AI/MXNet/IO.pm view on Meta::CPAN
# Create an io iterator by handle.
func _make_io_iterator($handle)
{
my ($iter_name, $desc,
$arg_names, $arg_types, $arg_descs
) = @{ check_call(AI::MXNetCAPI::DataIterGetIterInfo($handle)) };
my $param_str = build_param_doc($arg_names, $arg_types, $arg_descs);
my $doc_str = "$desc\n\n"
."$param_str\n"
."name : string, required.\n"
." Name of the resulting data iterator.\n\n"
."Returns\n"
."-------\n"
."iterator: DataIter\n"
." The result iterator.";
my $iter = sub {
my $class = shift;
my (@args, %kwargs);
if(@_ and ref $_[-1] eq 'HASH')
{
%kwargs = %{ pop(@_) };
}
@args = @_;
Carp::confess("$iter_name can only accept keyword arguments")
if @args;
lib/AI/MXNet/Image.pm view on Meta::CPAN
use AI::MXNet::Base;
use AI::MXNet::Function::Parameters;
=head1 NAME
AI::MXNet:Image - Read individual image files and perform augmentations.
=cut
=head2 imdecode
Decode an image from string. Requires OpenCV to work.
Parameters
----------
$buf : str, array ref, pdl, ndarray
Binary image data.
:$flag : int
0 for grayscale. 1 for colored.
:$to_rgb : int
0 for BGR format (OpenCV default). 1 for RGB format (MXNet default).
:$out : NDArray
lib/AI/MXNet/Image.pm view on Meta::CPAN
{
($w, $h) = (($w*$sh)/$h, $sh);
}
if($sw < $w)
{
($w, $h) = ($sw, ($h*$sw)/$w);
}
return (int($w), int($h));
}
=head2 resize_short
Resize shorter edge to the size.
Parameters:
-----------
AI::MXNet::NDArray $src
Int $size
Int $interp=2
Returns:
--------
AI::MXNet::NDArray $resized_image
=cut
method resize_short(AI::MXNet::NDArray $src, Int $size, Int $interp=2)
{
my ($new_h, $new_w);
my ($h, $w) = @{ $src->shape };
if($h > $w)
{
($new_h, $new_w) = ($size*$h/$w, $size);
}
else
{
($new_h, $new_w) = ($size, $size*$w/$h);
}
return AI::MXNet::NDArray->_cvimresize($src, $new_w, $new_h, { interp=>$interp });
}
=head2 fixed_crop
Crop src at fixed location, and (optionally) resize it to the size.
Parameters:
-----------
AI::MXNet::NDArray $src
Int $x0
Int $y0
Int $w
Int $h
Maybe[Shape] $size=
Int $interp=2
lib/AI/MXNet/Image.pm view on Meta::CPAN
Returns:
--------
AI::MXNet::NDArray $cropped_image
=cut
method fixed_crop(AI::MXNet::NDArray $src, Int $x0, Int $y0, Int $w, Int $h, Maybe[Shape] $size=, Int $interp=2)
{
my $out = AI::MXNet::NDArray->crop($src, { begin=>[$y0, $x0, 0], end=>[$y0+$h, $x0+$w, $src->shape->[2]] });
if(defined $size and join(',', $w, $h) ne join(',', @{ $size }))
{
$out = AI::MXNet::NDArray->_cvimresize($out, @{ $size }, { interp=>$interp });
}
return $out;
}
=head2 random_crop
Randomly crop src with size. Upsample result if src is smaller than the size.
Parameters:
-----------
AI::MXNet::NDArray $src
Shape $size=
Int $interp=2
Returns:
--------
($cropped_image, [$x0, $y0, $new_w, $new_h])
lib/AI/MXNet/Image.pm view on Meta::CPAN
my $x0 = int(rand($w - $new_w + 1));
my $y0 = int(rand($h - $new_h + 1));
my $out = __PACKAGE__->fixed_crop($src, $x0, $y0, $new_w, $new_h, $size, $interp);
return ($out, [$x0, $y0, $new_w, $new_h]);
}
=head2 center_crop
Randomly crop src with size around the center. Upsample result if src is smaller than the size.
Parameters:
-----------
AI::MXNet::NDArray $src
Shape $size=
Int $interp=2
Returns:
--------
($cropped_image, [$x0, $y0, $new_w, $new_h])
lib/AI/MXNet/Image.pm view on Meta::CPAN
assert($new_w <= $w and $new_h <= $h);
my $x0 = int(rand($w - $new_w + 1));
my $y0 = int(rand($h - $new_h + 1));
my $out = __PACKAGE__->fixed_crop($src, $x0, $y0, $new_w, $new_h, $size, $interp);
return ($out, [$x0, $y0, $new_w, $new_h]);
}
=head2 ResizeAug
Makes "resize shorter edge to size augumenter" closure.
Parameters:
-----------
Shape $size
Int $interp=2
Returns:
--------
CodeRef that accepts AI::MXNet::NDArray $src as input
and returns [__PACKAGE__->resize_short($src, $size, $interp)]
=cut
method ResizeAug(Shape $size, Int $interp=2)
{
my $aug = sub {
my $src = shift;
return [__PACKAGE__->resize_short($src, $size, $interp)];
};
return $aug;
}
=head2 RandomCropAug
Makes "random crop augumenter" closure.
Parameters:
-----------
lib/AI/MXNet/Image.pm view on Meta::CPAN
return $aug;
}
=head2 CreateAugmenter
Create augumenter list
Parameters:
-----------
Shape :$data_shape,
Bool :$resize=0,
Bool :$rand_crop=0,
Bool :$rand_resize=0,
Bool :$rand_mirror=0,
Maybe[Num|PDL] :$mean=,
Maybe[Num|PDL] :$std=,
Num :$brightness=0,
Num :$contrast=0,
Num :$saturation=0,
Num :$pca_noise=0,
Int :$inter_method=2
=cut
method CreateAugmenter(
Shape :$data_shape,
Bool :$resize=0,
Bool :$rand_crop=0,
Bool :$rand_resize=0,
Bool :$rand_mirror=0,
Maybe[Num|PDL] :$mean=,
Maybe[Num|PDL] :$std=,
Num :$brightness=0,
Num :$contrast=0,
Num :$saturation=0,
Num :$pca_noise=0,
Int :$inter_method=2
)
{
my @auglist;
if($resize > 0)
{
push @auglist, __PACKAGE__->ResizeAug($resize, $inter_method);
}
my $crop_size = [$data_shape->[2], $data_shape->[1]];
if($rand_resize)
{
assert($rand_crop);
push @auglist, __PACKAGE__->RandomSizedCropAug($crop_size, 0.3, [3.0/4.0, 4.0/3.0], $inter_method);
}
elsif($rand_crop)
{
push @auglist, __PACKAGE__->RandomCropAug($crop_size, $inter_method);
}
else
{
lib/AI/MXNet/Image.pm view on Meta::CPAN
my $label = AI::MXNet::NDArray->array([@line[1..@line-2]]);
my $key = $line[0];
$imglist{$key} = [$label, $line[-1]];
push @imgkeys, $key;
}
$self->imglist(\%imglist);
}
elsif(ref $self->imglist eq 'ARRAY')
{
print("loading image list...\n");
my %result;
my $index = 1;
for my $img (@{ $self->imglist })
{
my $key = $index++;
my $label;
if(not ref $img->[0])
{
$label = AI::MXNet::NDArray->array([$img->[0]]);
}
else
{
$label = AI::MXNet::NDArray->array($img->[0]);
$result{$key} = [$label, $img->[1]];
push @imgkeys, $key;
}
}
$self->imglist(\%result);
}
assert(@{ $self->data_shape } == 3 and $self->data_shape->[0] == 3);
$self->provide_data([
AI::MXNet::DataDesc->new(
name => $self->data_name,
shape => [$self->batch_size, @{ $self->data_shape }]
)
]);
if($self->label_width > 1)
{
lib/AI/MXNet/Image.pm view on Meta::CPAN
}
if(defined $self->aug_list or defined $self->kwargs)
{
$self->aug_list(AI::MXNet::Image->CreateAugmenter(data_shape => $self->data_shape, %{ $self->kwargs//{} }));
}
else
{
$self->aug_list([]);
}
$self->cur(0);
$self->reset();
}
method reset()
{
if($self->shuffle)
{
@{ $self->seq } = List::Util::shuffle(@{ $self->seq });
}
if(defined $self->imgrec)
{
$self->imgrec->reset;
}
$self->cur(0);
}
method next_sample()
{
if(defined $self->seq)
{
return undef if($self->cur >= @{ $self->seq });
my $idx = $self->seq->[$self->cur];
lib/AI/MXNet/Initializer.pm view on Meta::CPAN
*{"$orig_name"} = sub { shift; $self->new(@_) };
*InitDesc = sub { shift; AI::MXNet::InitDesc->new(@_) };
}
}
=head2 init
Parameters
----------
$desc : AI::MXNet::InitDesc|str
a name of corresponding ndarray
or the object that describes the initializer.
$arr : AI::MXNet::NDArray
an ndarray to be initialized.
=cut
method call(Str|AI::MXNet::InitDesc $desc, AI::MXNet::NDArray $arr)
{
return $self->_legacy_init($desc, $arr) unless blessed $desc;
my $init = $desc->attrs->{ __init__ };
if($init)
lib/AI/MXNet/Initializer.pm view on Meta::CPAN
my $shape = $arr->shape;
my $size = $arr->size;
my $f = pceil($shape->[3] / 2)->at(0);
my $c = (2 * $f - 1 - $f % 2) / (2 * $f);
for my $i (0..($size-1))
{
my $x = $i % $shape->[3];
my $y = ($i / $shape->[3]) % $shape->[2];
$weight->index($i) .= (1 - abs($x / $f - $c)) * (1 - abs($y / $f - $c));
}
$arr .= $weight->reshape(reverse @{ $shape });
}
method _init_loc_bias($name, $arr)
{
confess("assert error shape[0] == 6")
unless $arr->shape->[0] == 6;
$arr .= [1.0, 0, 0, 0, 1.0, 0];
}
method _init_zero($name, $arr)
lib/AI/MXNet/Initializer.pm view on Meta::CPAN
*slice = *call;
=head1 NAME
AI::MXNet::Mixed - A container for multiple initializer patterns.
=cut
=head2 new
patterns: array ref of str
array ref of regular expression patterns to match parameter names.
initializers: array ref of AI::MXNet::Initializer objects.
array ref of Initializers corresponding to the patterns.
=cut
package AI::MXNet::Mixed;
use Mouse;
extends 'AI::MXNet::Initializer';
has "map" => (is => "rw", init_arg => undef);
has "patterns" => (is => "ro", isa => 'ArrayRef[Str]');
has "initializers" => (is => "ro", isa => 'ArrayRef[AI::MXnet::Initializer]');
lib/AI/MXNet/Initializer.pm view on Meta::CPAN
my ($u, $s, $v) = svd($tmp);
my $q;
if(join(',', @{ $u->shape->unpdl }) eq join(',', @{ $tmp->shape->unpdl }))
{
$q = $u;
}
else
{
$q = $v;
}
$q = $self->scale * $q->reshape(reverse(@shape));
$arr .= $q;
}
*slice = *call;
__PACKAGE__->register;
=head1 NAME
AI::MXNet::Xavier - Initialize the weight with Xavier or similar initialization scheme.
=cut
lib/AI/MXNet/Initializer.pm view on Meta::CPAN
my $shape = $arr->shape;
my $size = $arr->size;
my $f = pceil($shape->[3] / 2)->at(0);
my $c = (2 * $f - 1 - $f % 2) / (2 * $f);
for my $i (0..($size-1))
{
my $x = $i % $shape->[3];
my $y = ($i / $shape->[3]) % $shape->[2];
$weight->index($i) .= (1 - abs($x / $f - $c)) * (1 - abs($y / $f - $c));
}
$arr .= $weight->reshape(reverse @{ $shape });
}
__PACKAGE__->register;
package AI::MXNet::LSTMBias;
=head1 NAME
AI::MXNet::LSTMBias - Custom initializer for LSTM cells.
=cut
lib/AI/MXNet/LRScheduler.pm view on Meta::CPAN
fallback => 1;
=head1 NAME
AI::MXNet::LRScheduler - The adaptive scheduler of the learning rate.
=cut
=head1 DESCRIPTION
Learning rate scheduler, which adaptively changes the learning rate based on the
progress.
=cut
=head2 new
base_lr : float (optional, default 0.01)
the initial learning rate
=cut
has 'base_lr' => (is => 'rw', isa => 'Num', default => 0.01);
=head2 call
Call to schedule current learning rate
The training progress is presented by num_update, which can be roughly
viewed as the number of minibatches executed so far. Its value is
non-decreasing, and increases at most by one.
The exact value is the upper bound of the number of updates applied to
a weight/index
See more details in https://github.com/dmlc/mxnet/issues/625
Parameters
----------
lib/AI/MXNet/Metric.pm view on Meta::CPAN
[shift->get_name_value()]
)->Purity(1)->Deepcopy(1)->Terse(1)->Dump
}, fallback => 1;
has 'name' => (is => 'rw', isa => 'Str');
has 'num' => (is => 'rw', isa => 'Int');
has 'num_inst' => (is => 'rw', isa => 'Maybe[Int|ArrayRef[Int]]');
has 'sum_metric' => (is => 'rw', isa => 'Maybe[Num|ArrayRef[Num]]');
sub BUILD
{
shift->reset;
}
method update($label, $pred)
{
confess('NotImplemented');
}
method reset()
{
if(not defined $self->num)
{
$self->num_inst(0);
$self->sum_metric(0);
}
else
{
$self->num_inst([(0) x $self->num]);
$self->sum_metric([(0) x $self->num]);
lib/AI/MXNet/Metric.pm view on Meta::CPAN
}
method update(ArrayRef[AI::MXNet::NDArray] $labels, ArrayRef[AI::MXNet::NDArray] $preds)
{
for my $metric (@{ $self->metrics })
{
$metric->update($labels, $preds);
}
}
method reset()
{
for my $metric (@{ $self->metrics })
{
$metric->reset;
}
}
method get()
{
my $names = [];
my $results = [];
for my $metric (@{ $self->metrics })
{
my ($name, $result) = $metric->get;
$name = [$name] unless ref $name;
$result = [$result] unless ref $result;
push @$names, @$name;
push @$results, @$result;
}
return ($names, $results);
}
########################
# CLASSIFICATION METRICS
########################
package AI::MXNet::Accuracy;
use Mouse;
use AI::MXNet::Base;
lib/AI/MXNet/Metric.pm view on Meta::CPAN
AI::MXNet::Metric::check_label_shapes($labels, $preds);
my ($loss, $num) = (0, 0);
zip(sub {
my ($label, $pred) = @_;
my $label_shape = $label->shape;
my $pred_shape = $pred->shape;
assert(
(product(@{ $label_shape }) == product(@{ $pred_shape })/$pred_shape->[-1]),
"shape mismatch: (@$label_shape) vs. (@$pred_shape)"
);
$label = $label->as_in_context($pred->context)->reshape([$label->size]);
$pred = AI::MXNet::NDArray->pick($pred, $label->astype('int32'), { axis => $self->axis });
if(defined $self->ignore_label)
{
my $ignore = ($label == $self->ignore_label);
$num -= $ignore->sum->asscalar;
$pred = $pred*(1-$ignore) + $ignore;
}
$loss -= $pred->maximum(1e-10)->log->sum->asscalar;
$num += $pred->size;
}, $labels, $preds);
lib/AI/MXNet/Metric.pm view on Meta::CPAN
method update(ArrayRef[AI::MXNet::NDArray] $labels, ArrayRef[AI::MXNet::NDArray] $preds)
{
AI::MXNet::Metric::check_label_shapes($labels, $preds);
zip(sub {
my ($label, $pred) = @_;
$label = $label->aspdl;
$pred = $pred->aspdl;
if($label->ndims == 1)
{
$label = $label->reshape(1, $label->shape->at(0));
}
$self->sum_metric($self->sum_metric + ($label - $pred)->abs->avg);
$self->num_inst($self->num_inst + 1);
}, $labels, $preds);
}
# Calculate Mean Squared Error loss
package AI::MXNet::MSE;
use Mouse;
use AI::MXNet::Base;
lib/AI/MXNet/Metric.pm view on Meta::CPAN
method update(ArrayRef[AI::MXNet::NDArray] $labels, ArrayRef[AI::MXNet::NDArray] $preds)
{
AI::MXNet::Metric::check_label_shapes($labels, $preds);
zip(sub {
my ($label, $pred) = @_;
$label = $label->aspdl;
$pred = $pred->aspdl;
if($label->ndims == 1)
{
$label = $label->reshape(1, $label->shape->at(0));
}
$self->sum_metric($self->sum_metric + (($label - $pred)**2)->avg);
$self->num_inst($self->num_inst + 1);
}, $labels, $preds);
}
# Calculate Root Mean Squred Error loss
package AI::MXNet::RMSE;
use Mouse;
use AI::MXNet::Base;
lib/AI/MXNet/Metric.pm view on Meta::CPAN
method update(ArrayRef[AI::MXNet::NDArray] $labels, ArrayRef[AI::MXNet::NDArray] $preds)
{
AI::MXNet::Metric::check_label_shapes($labels, $preds);
zip(sub {
my ($label, $pred) = @_;
$label = $label->aspdl;
$pred = $pred->aspdl;
if($label->ndims == 1)
{
$label = $label->reshape(1, $label->shape->at(0));
}
$self->sum_metric($self->sum_metric + sqrt((($label - $pred)**2)->avg));
$self->num_inst($self->num_inst + 1);
}, $labels, $preds);
}
# Calculate Cross Entropy loss
package AI::MXNet::CrossEntropy;
use Mouse;
use AI::MXNet::Base;
lib/AI/MXNet/Module.pm view on Meta::CPAN
$mod->params_initialized(1);
if($load_optimizer_states)
{
$mod->_p->_preload_opt_states(sprintf('%s-%04d.states', $prefix, $epoch));
}
return $mod;
}
=head2 save_checkpoint
Save current progress to a checkpoint.
Use mx->callback->module_checkpoint as epoch_end_callback to save during training.
Parameters
----------
prefix : str
The file prefix to checkpoint to
epoch : int
The current epoch number
save_optimizer_states : bool
Whether to save optimizer states for later training
lib/AI/MXNet/Module.pm view on Meta::CPAN
{
if(defined $symbol)
{
$symbol->save("$prefix-symbol.json");
}
my $param_name = sprintf('%s-%04d.params', $prefix, $epoch);
$self->save_params($param_name, $arg_params, $aux_params);
AI::MXNet::Logging->info('Saved checkpoint to "%s"', $param_name);
}
# Internal function to reset binded state.
method _reset_bind()
{
$self->binded(0);
$self->_p->_exec_group(undef);
$self->_p->_data_shapes(undef);
$self->_p->_label_shapes(undef);
}
method data_names()
{
return $self->_p->_data_names;
lib/AI/MXNet/Module.pm view on Meta::CPAN
# just in case the cached array is just the target itself
if($cache_arr->handle ne $arr->handle)
{
$cache_arr->copyto($arr);
}
}
else
{
if(not $allow_missing)
{
confess("$name is not presented");
}
if(defined $initializer)
{
&{$initializer}($name, $arr);
}
}
}
else
{
&{$initializer}($name, $arr) if defined $initializer;
lib/AI/MXNet/Module.pm view on Meta::CPAN
Default is 1. Whether the executors should be bind for training.
:$inputs_need_grad : bool
Default is 0. Whether the gradients to the input data need to be computed.
Typically this is not needed. But this might be needed when implementing composition
of modules.
:$force_rebind : bool
Default is 0. This function does nothing if the executors are already
binded. But with this 1, the executors will be forced to rebind.
:$shared_module : Module
Default is undef. This is used in bucketing. When not undef, the shared module
essentially corresponds to a different bucket -- a module with different symbol
but with the same sets of parameters (e.g. unrolled RNNs with different lengths).
=cut
method bind(
ArrayRef[AI::MXNet::DataDesc|NameShape] :$data_shapes,
Maybe[ArrayRef[AI::MXNet::DataDesc|NameShape]] :$label_shapes=,
Bool :$for_training=1,
Bool :$inputs_need_grad=0,
Bool :$force_rebind=0,
Maybe[AI::MXNet::Module] :$shared_module=,
GradReq|HashRef[GradReq]|ArrayRef[GradReq] :$grad_req='write',
Maybe[ArrayRef[Str]] :$state_names=$self->_p->_state_names
)
{
# force rebinding is typically used when one want to switch from
# training to prediction phase.
if($force_rebind)
{
$self->_reset_bind();
}
if($self->binded)
{
$self->logger->warning('Already binded, ignoring bind()');
return;
}
$self->for_training($for_training);
$self->inputs_need_grad($inputs_need_grad);
$self->binded(1);
$self->_p->_grad_req($grad_req);
lib/AI/MXNet/Module.pm view on Meta::CPAN
my %aux_params;
@aux_params{ @{ $self->_p->_aux_names } } = @aux_arrays;
$self->_p->_aux_params(\%aux_params);
}
if($shared_module and $shared_module->optimizer_initialized)
{
$self->borrow_optimizer($shared_module)
}
}
=head2 reshape
Reshape the module for new input shapes.
Parameters
----------
:$data_shapes : ArrayRef[AI::MXNet::DataDesc]
Typically is $data_iter->provide_data.
:$label_shapes= : Maybe[ArrayRef[AI::MXNet::DataDesc]]
Typically is $data_iter->provide_label.
=cut
method reshape(
ArrayRef[AI::MXNet::DataDesc|NameShape] :$data_shapes,
Maybe[ArrayRef[AI::MXNet::DataDesc|NameShape]] :$label_shapes=
)
{
assert($self->binded);
($data_shapes, $label_shapes) = $self->_parse_data_desc(
$self->data_names, $self->label_names, $data_shapes, $label_shapes
);
$self->_p->_data_shapes($data_shapes);
$self->_p->_label_shapes($label_shapes);
$self->_p->_exec_group->reshape($self->_p->_data_shapes, $self->_p->_label_shapes);
}
method init_optimizer(
Str|AI::MXNet::KVStore :$kvstore='local',
Optimizer :$optimizer='sgd',
HashRef :$optimizer_params={ learning_rate => 0.01 },
Bool :$force_init=0
)
{
assert($self->binded and $self->params_initialized);
lib/AI/MXNet/Module.pm view on Meta::CPAN
my ($kvstore, $update_on_kvstore) = _create_kvstore(
$kvstore,
scalar(@{$self->_p->_context}),
$self->_p->_arg_params
);
my $batch_size = $self->_p->_exec_group->_p->batch_size;
if($kvstore and $kvstore->type =~ /dist/ and $kvstore->type =~ /_sync/)
{
$batch_size *= $kvstore->num_workers;
}
my $rescale_grad = 1/$batch_size;
if(not blessed $optimizer)
{
my %idx2name;
if($update_on_kvstore)
{
@idx2name{ 0..@{$self->_p->_exec_group->param_names}-1 } = @{$self->_p->_exec_group->param_names};
}
else
{
for my $k (0..@{$self->_p->_context}-1)
{
@idx2name{ map { $_ + $k } 0..@{$self->_p->_exec_group->param_names}-1 } = @{$self->_p->_exec_group->param_names};
}
}
if(not exists $optimizer_params->{rescale_grad})
{
$optimizer_params->{rescale_grad} = $rescale_grad;
}
$optimizer = AI::MXNet::Optimizer->create(
$optimizer,
sym => $self->symbol,
param_idx2name => \%idx2name,
%{ $optimizer_params }
);
if($optimizer->rescale_grad != $rescale_grad)
{
AI::MXNet::Logging->warning(
"Optimizer created manually outside Module but rescale_grad "
."is not normalized to 1.0/batch_size/num_workers (%s vs. %s). "
."Is this intended?",
$optimizer->rescale_grad, $rescale_grad
);
}
}
$self->_p->_optimizer($optimizer);
$self->_p->_kvstore($kvstore);
$self->_p->_update_on_kvstore($update_on_kvstore);
$self->_p->_updater(undef);
if($kvstore)
lib/AI/MXNet/Module.pm view on Meta::CPAN
elsif($data_batch->can('label') and $data_batch->label)
{
$new_lshape = [];
zip(sub {
my ($i, $j) = @_;
push @{ $new_lshape }, AI::MXNet::DataDesc->new(
$i->name, $j->shape, $i->dtype, $i->layout
);
}, $self->label_shapes, $data_batch->label);
}
$self->reshape(data_shapes => $new_dshape, label_shapes => $new_lshape);
}
$self->_p->_exec_group->forward($data_batch, $is_train);
}
method backward(Maybe[AI::MXNet::NDArray|ArrayRef[AI::MXNet::NDArray]] $out_grads=)
{
assert($self->binded and $self->params_initialized);
$self->_p->_exec_group->backward($out_grads);
}
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
}
else
{
$self->_check_names_match($label_names, [], 'label', 0);
}
return ($data_shapes, $label_shapes);
}
=head1 DESCRIPTION
The base class of a modules. A module represents a computation component. The design
purpose of a module is that it abstract a computation "machine", that one can run forward,
backward, update parameters, etc. We aim to make the APIs easy to use, especially in the
case when we need to use imperative API to work with multiple modules (e.g. stochastic
depth network).
A module has several states:
- Initial state. Memory is not allocated yet, not ready for computation yet.
- Binded. Shapes for inputs, outputs, and parameters are all known, memory allocated,
ready for computation.
- Parameter initialized. For modules with parameters, doing computation before initializing
the parameters might result in undefined outputs.
- Optimizer installed. An optimizer can be installed to a module. After this, the parameters
of the module can be updated according to the optimizer after gradients are computed
(forward-backward).
In order for a module to interact with others, a module should be able to report the
following information in its raw stage (before binded)
- data_names: array ref of string indicating the names of required data.
- output_names: array ref of string indicating the names of required outputs.
And also the following richer information after binded:
- state information
- binded: bool, indicating whether the memory buffers needed for computation
has been allocated.
- for_training: whether the module is binded for training (if binded).
- params_initialized: bool, indicating whether the parameters of this modules
has been initialized.
- optimizer_initialized: bool, indicating whether an optimizer is defined
and initialized.
- inputs_need_grad: bool, indicating whether gradients with respect to the
input data is needed. Might be useful when implementing composition of modules.
- input/output information
- data_shapes: am array ref of [name, shape]. In theory, since the memory is allocated,
we could directly provide the data arrays. But in the case of data parallelization,
the data arrays might not be of the same shape as viewed from the external world.
- label_shapes: an array ref of [name, shape]. This might be [] if the module does
not need labels (e.g. it does not contains a loss function at the top), or a module
is not binded for training.
- output_shapes: an array ref of [name, shape] for outputs of the module.
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
- setup
- bind(): prepare environment for computation.
- init_optimizer(): install optimizer for parameter updating.
- computation
- forward(data_batch): forward operation.
- backward(out_grads=): backward operation.
- update(): update parameters according to installed optimizer.
- get_outputs(): get outputs of the previous forward operation.
- get_input_grads(): get the gradients with respect to the inputs computed
in the previous backward operation.
- update_metric(metric, labels): update performance metric for the previous forward
computed results.
- other properties (mostly for backward compatability)
- symbol: the underlying symbolic graph for this module (if any)
This property is not necessarily constant. For example, for AI::MXNet::Module::Bucketing,
this property is simply the *current* symbol being used. For other modules,
this value might not be well defined.
When those intermediate-level API are implemented properly, the following
high-level API will be automatically available for a module:
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
Parameters
----------
$eval_data : AI::MXNet::DataIter
$eval_metric : AI::MXNet::EvalMetric
:$num_batch= : Maybe[Int]
Number of batches to run. Default is undef, indicating run until the AI::MXNet::DataIter
finishes.
:$batch_end_callback= : Maybe[Callback]
Could also be a array ref of functions.
:$reset=1 : Bool
Default 1, indicating whether we should reset $eval_data before starting
evaluating.
$epoch=0 : Int
Default is 0. For compatibility, this will be passed to callbacks (if any). During
training, this will correspond to the training epoch number.
=cut
method score(
AI::MXNet::DataIter $eval_data,
EvalMetric $eval_metric,
Maybe[Int] :$num_batch=,
Maybe[Callback]|ArrayRef[Callback] :$batch_end_callback=,
Maybe[Callback]|ArrayRef[Callback] :$score_end_callback=,
Bool :$reset=1,
Int :$epoch=0
)
{
assert($self->binded and $self->params_initialized);
$eval_data->reset if $reset;
if(not blessed $eval_metric or not $eval_metric->isa('AI::MXNet::EvalMetric'))
{
$eval_metric = AI::MXNet::Metric->create($eval_metric);
}
$eval_metric->reset();
my $actual_num_batch = 0;
my $nbatch = 0;
while(my $eval_batch = <$eval_data>)
{
last if (defined $num_batch and $nbatch == $num_batch);
$self->forward($eval_batch, is_train => 0);
$self->update_metric($eval_metric, $eval_batch->label);
if (defined $batch_end_callback)
{
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
=head2 iter_predict
Iterate over predictions.
Parameters
----------
$eval_data : AI::MXNet::DataIter
:$num_batch= : Maybe[Int]
Default is undef, indicating running all the batches in the data iterator.
:$reset=1 : bool
Default is 1, indicating whether we should reset the data iter before start
doing prediction.
=cut
method iter_predict(AI::MXNet::DataIter $eval_data, Maybe[Int] :$num_batch=, Bool :$reset=1)
{
assert($self->binded and $self->params_initialized);
if($reset)
{
$eval_data->reset;
}
my $nbatch = 0;
my @out;
while(my $eval_batch = <$eval_data>)
{
last if defined $num_batch and $nbatch == $num_batch;
$self->forward($eval_batch, is_train => 0);
my $pad = $eval_batch->pad;
my $outputs = [
map { $_->slice([0, $_->shape->[0] - ($pad//0) - 1]) } @{ $self->get_outputs() }
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
Run prediction and collect the outputs.
Parameters
----------
$eval_data : AI::MXNet::DataIter
:$num_batch= : Maybe[Int]
Default is undef, indicating running all the batches in the data iterator.
:$merge_batches=1 : Bool
Default is 1.
:$reset=1 : Bool
Default is 1, indicating whether we should reset the data iter before start
doing prediction.
:$always_output_list=0 : Bool
Default is 0, see the doc for return values.
Returns
-------
When $merge_batches is 1 (by default), the return value will be an array ref
[$out1, $out2, $out3] where each element is concatenation of the outputs for
all the mini-batches. If $always_output_list` also is 0 (by default),
then in the case of a single output, $out1 is returned in stead of [$out1].
When $merge_batches is 0, the return value will be a nested array ref like
[[$out1_batch1, $out2_batch1], [$out1_batch2], ...]. This mode is useful because
in some cases (e.g. bucketing), the module does not necessarily produce the same
number of outputs.
The objects in the results are AI::MXNet::NDArray`s. If you need to work with pdl array,
just call ->aspdl() on each AI::MXNet::NDArray.
=cut
method predict(
AI::MXNet::DataIter $eval_data,
Maybe[Int] :$num_batch=, Bool :$merge_batches=1, Bool :$reset=1, Bool :$always_output_list=0
)
{
assert($self->binded and $self->params_initialized);
$eval_data->reset() if $reset;
my @output_list;
my $nbatch = 0;
while(my $eval_batch = <$eval_data>)
{
last if defined $num_batch and $nbatch == $num_batch;
$self->forward($eval_batch, is_train => 0);
my $pad = $eval_batch->pad;
my $outputs = [map { $_->slice([0, $_->shape->[0]-($pad//0)-1])->copy } @{ $self->get_outputs }];
push @output_list, $outputs;
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
:$allow_missing=0 : Bool
Default is 0. Indicates whether we allow missing parameters when $arg_params
and $aux_params are not undefined. If this is 1, then the missing parameters
will be initialized via the $initializer.
:$force_rebind=0 : Bool
Default is 0. Whether to force rebinding the executors if already binded.
:$force_init=0 : Bool
Default is 0. Indicates whether we should force initialization even if the
parameters are already initialized.
:$begin_epoch=0 : Int
Default is 0. Indicates the starting epoch. Usually, if we are resuming from a
checkpoint saved at a previous training phase at epoch N, then we should specify
this value as N+1.
:$num_epoch : Int
Number of epochs for the training.
=cut
method fit(
AI::MXNet::DataIter $train_data,
Maybe[AI::MXNet::DataIter] :$eval_data=,
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
}
$eval_metric = AI::MXNet::Metric->create($eval_metric)
unless blessed $eval_metric;
################################################################################
# training loop
################################################################################
for my $epoch ($begin_epoch..$num_epoch-1)
{
my $tic = time;
$eval_metric->reset;
my $nbatch = 0;
my $end_of_batch = 0;
my $next_data_batch = <$train_data>;
while(not $end_of_batch)
{
my $data_batch = $next_data_batch;
$monitor->tic if $monitor;
$self->forward_backward($data_batch);
$self->update;
$next_data_batch = <$train_data>;
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
{
for my $callback (@{ _as_list($epoch_end_callback) })
{
&{$callback}($epoch, $self->get_symbol, $arg_params, $aux_params);
}
}
#----------------------------------------
# evaluation on validation set
if(defined $eval_data)
{
my $res = $self->score(
$eval_data,
$validation_metric,
score_end_callback => $eval_end_callback,
batch_end_callback => $eval_batch_end_callback,
epoch => $epoch
);
#TODO: pull this into default
while(my ($name, $val) = each %{ $res })
{
$self->logger->info('Epoch[%d] Validation-%s=%f', $epoch, $name, $val);
}
}
# end of 1 epoch, reset the data-iter for another epoch
$train_data->reset;
}
}
################################################################################
# Symbol information
################################################################################
=head2 get_symbol
The symbol used by this module.
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
=head2 get_states
The states from all devices
Parameters
----------
$merge_multi_context=1 : Bool
Default is true (1). In the case when data-parallelism is used, the states
will be collected from multiple devices. A true value indicate that we
should merge the collected results so that they look like from a single
executor.
Returns
-------
If $merge_multi_context is 1, it is like [$out1, $out2]. Otherwise, it
is like [[$out1_dev1, $out1_dev2], [$out2_dev1, $out2_dev2]]. All the output
elements are AI::MXNet::NDArray.
=cut
method get_states(Bool $merge_multi_context=1)
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
----------
$mon : AI::MXNet::Monitor
=cut
method install_monitor(AI::MXNet::Monitor $mon) { confess("NotImplemented") }
=head2 prepare
Prepare the module for processing a data batch.
Usually involves switching a bucket and reshaping.
Parameters
----------
$data_batch : AI::MXNet::DataBatch
=cut
method prepare(AI::MXNet::DataBatch $data_batch){}
################################################################################
# Computations
################################################################################
=head2 forward
Forward computation. It supports data batches with different shapes, such as
different batch sizes or different image sizes.
If reshaping of data batch relates to modification of symbol or module, such as
changing image layout ordering or switching from training to predicting, module
rebinding is required.
Parameters
----------
$data_batch : DataBatch
Could be anything with similar API implemented.
:$is_train= : Bool
Default is undef, which means is_train takes the value of $self->for_training.
=cut
lib/AI/MXNet/Module/Base.pm view on Meta::CPAN
Default is 1. Whether the executors should be bind for training.
:$inputs_need_grad=0 : Bool
Default is 0. Whether the gradients to the input data need to be computed.
Typically this is not needed. But this might be needed when implementing composition
of modules.
:$force_rebind=0 : Bool
Default is 0. This function does nothing if the executors are already
binded. But with this as 1, the executors will be forced to rebind.
:$shared_module= : A subclass of AI::MXNet::Module::Base
Default is undef. This is used in bucketing. When not undef, the shared module
essentially corresponds to a different bucket -- a module with different symbol
but with the same sets of parameters (e.g. unrolled RNNs with different lengths).
:$grad_req='write' : Str|ArrayRef[Str]|HashRef[Str]
Requirement for gradient accumulation. Can be 'write', 'add', or 'null'
(defaults to 'write').
Can be specified globally (str) or for each argument (array ref, hash ref).
=cut
method bind(
ArrayRef[AI::MXNet::DataDesc] $data_shapes,
Maybe[ArrayRef[AI::MXNet::DataDesc]] :$label_shapes=,
lib/AI/MXNet/Module/Bucketing.pm view on Meta::CPAN
}
my $sym_gen = sub {
my $seq_len = shift;
my $data = mx->sym->Variable('data');
my $label = mx->sym->Variable('softmax_label');
my $embed = mx->sym->Embedding(
data => $data, input_dim => scalar(keys %$vocabulary),
output_dim => $num_embed, name => 'embed'
);
$stack->reset;
my ($outputs, $states) = $stack->unroll($seq_len, inputs => $embed, merge_outputs => 1);
my $pred = mx->sym->Reshape($outputs, shape => [-1, $num_hidden]);
$pred = mx->sym->FullyConnected(data => $pred, num_hidden => scalar(keys %$vocabulary), name => 'pred');
$label = mx->sym->Reshape($label, shape => [-1]);
$pred = mx->sym->SoftmaxOutput(data => $pred, label => $label, name => 'softmax');
return ($pred, ['data'], ['softmax_label']);
};
my $contexts;
if(defined $gpus)
lib/AI/MXNet/Module/Bucketing.pm view on Meta::CPAN
$self->_fixed_param_names([]) unless defined $original_params->{fixed_param_names};
$self->_state_names([]) unless defined $original_params->{state_names};
$self->_params_dirty(0);
my ($symbol, $data_names, $label_names) = &{$self->_sym_gen}($self->_default_bucket_key);
$self->_check_input_names($symbol, $data_names//[], "data", 1);
$self->_check_input_names($symbol, $label_names//[], "label", 0);
$self->_check_input_names($symbol, $self->_state_names, "state", 1);
$self->_check_input_names($symbol, $self->_fixed_param_names, "fixed_param", 1);
}
method _reset_bind()
{
$self->binded(0);
$self->_buckets({});
$self->_curr_module(undef);
$self->_curr_bucket_key(undef);
}
method data_names()
{
if($self->binded)
lib/AI/MXNet/Module/Bucketing.pm view on Meta::CPAN
method set_states(:$states=, :$value=)
{
assert($self->binded and $self->params_initialized);
$self->_curr_module->set_states(states => $states, value => $value);
}
=head2 bind
Binding for a AI::MXNet::Module::Bucketing means setting up the buckets and bind the
executor for the default bucket key. Executors corresponding to other keys are
binded afterwards with switch_bucket.
Parameters
----------
:$data_shapes : ArrayRef[AI::MXNet::DataDesc|NameShape]
This should correspond to the symbol for the default bucket.
:$label_shapes= : Maybe[ArrayRef[AI::MXNet::DataDesc|NameShape]]
This should correspond to the symbol for the default bucket.
:$for_training : Bool
Default is 1.
:$inputs_need_grad : Bool
Default is 0.
:$force_rebind : Bool
Default is 0.
:$shared_module : AI::MXNet::Module::Bucketing
Default is undef. This value is currently not used.
:$grad_req : str, array ref of str, hash ref of str to str
Requirement for gradient accumulation. Can be 'write', 'add', or 'null'
lib/AI/MXNet/Module/Bucketing.pm view on Meta::CPAN
{
# in case we already initialized params, keep it
my ($arg_params, $aux_params);
if($self->params_initialized)
{
($arg_params, $aux_params) = $self->get_params;
}
# force rebinding is typically used when one want to switch from
# training to prediction phase.
$self->_reset_bind if $force_rebind;
if($self->binded)
{
$self->logger->warning('Already binded, ignoring bind()');
return;
}
assert((not defined $shared_module), 'shared_module for BucketingModule is not supported');
$self->for_training($for_training);
lib/AI/MXNet/Monitor.pm view on Meta::CPAN
Parameters
----------
interval : int
Number of batches between printing.
stat_func : function
a function that computes statistics of tensors.
Takes a NDArray and returns a NDArray. defaults to mean
absolute value |x|/size(x).
pattern : str
A regular expression specifying which tensors to monitor.
Only tensors with names that match name_pattern will be included.
For example, '.*weight|.*output' will print all weights and outputs;
'.*backward.*' will print all gradients.
=cut
has 'interval' => (is => 'ro', isa => 'Int', required => 1);
has 'stat_func' => (
is => 'ro',
isa => 'CodeRef',
default => sub {
lib/AI/MXNet/Monitor.pm view on Meta::CPAN
$_->wait_to_read for @{ $exe->aux_arrays };
}
$self->queue([]);
$self->activated(1);
}
$self->step($self->step + 1);
}
=head2 toc
End collecting for current batch and return results.
Call after computation of current batch.
Returns
-------
res : array ref of array refs with debug info
=cut
method toc()
{
return [] unless $self->activated;
for my $exe (@{ $self->exes })
{
$_->wait_to_read for @{ $exe->arg_arrays };
$_->wait_to_read for @{ $exe->aux_arrays };
}
lib/AI/MXNet/Monitor.pm view on Meta::CPAN
zip(sub {
my ($name, $array) = @_;
push @{ $self->queue }, [$self->step, $name, $self->stat_func->($array)];
}, $exe->_symbol->list_arguments, $exe->arg_arrays);
zip(sub {
my ($name, $array) = @_;
push @{ $self->queue }, [$self->step, $name, $self->stat_func->($array)];
}, $exe->_symbol->list_auxiliary_states, $exe->aux_arrays);
}
$self->activated(0);
my @res;
if($self->_sort)
{
@{ $self->queue } = sort { $a->[1] cmp $b->[1] } @{ $self->queue };
}
for my $q (@{ $self->queue })
{
my ($n, $k, $v_list) = @{ $q };
if(ref $v_list ne 'ARRAY')
{
$v_list = [$v_list];
lib/AI/MXNet/Monitor.pm view on Meta::CPAN
unless blessed($v) and $v->isa('AI::MXNet::NDArray');
if($v->size == 1)
{
$s .= $v->asscalar . "\t";
}
else
{
$s .= $v->aspdl . "\t";
}
}
push @res, [$n, $k, $s];
}
$self->queue([]);
return \@res;
}
=head2 toc_print
End collecting and print results
=cut
method toc_print()
{
my $res = $self->toc;
for my $r (@{ $res })
{
AI::MXNet::Logging->info('Batch: %7d %30s %s', @{ $r });
}
}
method Monitor(@args)
{
__PACKAGE__->new(@args % 2 ? ('interval', @args) : @args);
}
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
${$pdl->get_dataref} = $buf;
$pdl->upd_data;
return $pdl;
}
=head2 asmpdl
Returns copied PDL::Matrix objectt of current array.
Requires caller to "use PDL::Matrix" in user space.
Returns
-------
array : PDL::Matrix
A copy of array content.
=cut
method asmpdl()
{
my $dtype = $self->dtype;
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
$buf = pack("f*", map { AI::MXNetCAPI::_half_to_float($_) } unpack("S*", $buf));
}
${$pdl->get_dataref} = $buf;
$pdl->upd_data;
return $pdl;
}
=head2 _slice
Returns sliced NDArray that shares memory with the current one.
Parameters
----------
start : int
Starting index of slice.
stop : int
Finishing index of slice.
=cut
method _slice (
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
$self->handle,
$start,
$stop
)
);
return __PACKAGE__->new(handle => $handle, writable => $self->writable);
}
=head2 _at
Returns a sub NDArray that shares memory with current one.
Parameters
----------
idx : int
index of the sub array.
=cut
method _at(Index $idx)
{
my $handle = check_call(
AI::MXNetCAPI::NDArrayAt(
$self->handle, $idx >=0 ? $idx : $self->shape->[0] + $idx
)
);
return __PACKAGE__->new(handle => $handle, writable => $self->writable);
}
=head2 reshape
Returns a reshaped NDArray that shares the memory with current one.
One shape dimension can be -1. In this case, the value is inferred
from the length of the array and remaining dimensions.
Parameters
----------
new_shape : Shape
new shape of NDArray
=cut
method reshape(ArrayRef[Int] $new_shape)
{
my $i = -1;
my @inferred = map { $i++; $_ == -1 ? ($i) : () } @$new_shape;
assert((@inferred <= 1), 'Only one dimension can be inferred.');
if(@inferred)
{
$new_shape->[$inferred[0]] = product(@{ $self->shape })/product(map { abs($_) } @{ $new_shape });
}
my $handle = check_call(
AI::MXNetCAPI::NDArrayReshape(
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
Parameters
----------
source : int
Original position of the axes to move.
destination : int
Destination position for each of the original axes.
Returns
-------
result :NDArray
Array with moved axes.
Examples
--------
> $X = mx->nd->array([[1, 2, 3],
[4, 5, 6]]);
> print Dumper($X->moveaxis(0, 1)->shape)
> [3, 2]
=cut
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
}
@$cur_shape = ((1)x(@$shape - @$cur_shape), @$cur_shape);
my $cur_shape_arr = pdl($cur_shape);
my $broadcasting_axes = ($cur_shape_arr != pdl($shape))->which->unpdl;
if (grep { $cur_shape->[$_] != 1 } @$broadcasting_axes)
{
confess($err_str);
}
if(join(',',@$cur_shape) ne join(',',@{ $self->shape }))
{
return __PACKAGE__->SUPER::broadcast_to($self->reshape($cur_shape),{ shape => $shape });
}
else
{
return __PACKAGE__->SUPER::broadcast_to($self, { shape => $shape });
}
}
=head2 wait_to_read
Block until all pending write operations on the NDArray are finished.
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
{
check_call(AI::MXNetCAPI::NDArrayWaitToRead($self->handle));
}
=head2 shape
Get the shape of current NDArray.
Returns
-------
an array ref representing the shape of current ndarray
=cut
method shape()
{
return scalar(check_call(AI::MXNetCAPI::NDArrayGetShape($self->handle)));
}
=head2 size
Number of elements in the array.
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
);
}
=head2 dtype
The data type of current NDArray.
Returns
-------
a data type string ('float32', 'float64', 'float16', 'uint8', 'int32')
representing the data type of the ndarray.
'float32' is the default dtype for the ndarray class.
=cut
method dtype()
{
my $dtype = check_call(
AI::MXNetCAPI::NDArrayGetDType(
$self->handle
)
);
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
$dtype : Dtype
Returns
-------
$array : ndarray
A copy of the array content.
=cut
method astype(Dtype $dtype)
{
my $res = __PACKAGE__->empty($self->shape, ctx => $self->context, dtype => $dtype);
$self->copyto($res);
return $res;
}
=head2 as_in_context
Returns an NDArray in the target context.
If the array is already in that context, self is returned. Otherwise, a copy is
made.
Parameters
----------
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
return $self->copyto($context);
}
=head2 onehot_encode
One hot encoding indices into matrix out.
Parameters
----------
indices: NDArray
An NDArray containing indices of the categorical features.
out: NDArray
The result of the encoding.
Returns
-------
$out: NDArray
=cut
method onehot_encode(AI::MXNet::NDArray $indices, AI::MXNet::NDArray $out)
{
return __PACKAGE__->_onehot_encode($indices, $out, { out => $out });
}
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
lfn_scalar : function
function to be called if lhs is NDArray while rhs is numeric value
rfn_scalar : function
function to be called if lhs is numeric value while rhs is NDArray;
if none is provided, then the function is commutative, so rfn_scalar is equal to lfn_scalar
Returns
-------
out: NDArray
result array
=cut
sub _ufunc_helper
{
my ($lhs, $rhs, $fn_array, $lfn_scalar, $rfn_scalar, $reverse) = @_;
($rhs, $lhs) = ($lhs, $rhs) if $reverse and $rfn_scalar;
if(not ref $lhs)
{
if(not $rfn_scalar)
{
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
=cut
method concatenate(ArrayRef[AI::MXNet::NDArray] $arrays, Index :$axis=0, :$always_copy=1)
{
confess("no arrays provided") unless @$arrays > 0;
if(not $always_copy and @$arrays == 1)
{
return $arrays->[0];
}
my $shape_axis = $arrays->[0]->shape->[$axis];
my $shape_rest1 = [@{ $arrays->[0]->shape }[0..($axis-1)]];
my $shape_rest2 = [@{ $arrays->[0]->shape }[($axis+1)..(@{ $arrays->[0]->shape }-1)]];
my $dtype = $arrays->[0]->dtype;
my $i = 1;
for my $arr (@{ $arrays }[1..(@{ $arrays }-1)])
{
$shape_axis += $arr->shape->[$axis];
my $arr_shape_rest1 = [@{ $arr->shape }[0..($axis-1)]];
my $arr_shape_rest2 = [@{ $arr->shape }[($axis+1)..(@{ $arr->shape }-1)]];
confess("first array $arrays->[0] and $i array $arr do not match")
unless join(',',@$arr_shape_rest1) eq join(',',@$shape_rest1);
confess("first array $arrays->[0] and $i array $arr do not match")
unless join(',',@$arr_shape_rest2) eq join(',',@$shape_rest2);
confess("first array $arrays->[0] and $i array $arr dtypes do not match")
unless join(',',@$arr_shape_rest2) eq join(',',@$shape_rest2);
$i++;
}
my $ret_shape = [@$shape_rest1, $shape_axis, @$shape_rest2];
my $ret = __PACKAGE__->empty($ret_shape, ctx => $arrays->[0]->context, dtype => $dtype);
my $idx = 0;
my $begin = [(0)x@$ret_shape];
my $end = [@$ret_shape];
for my $arr (@$arrays)
{
if ($axis == 0)
{
$ret->slice([$idx,($idx+$arr->shape->[0]-1)]) .= $arr;
}
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
Loads ndarrays from a binary file.
You can also use Storable to do the job if you only work with Perl.
The advantage of load/save is the file is language agnostic.
This means the file saved using save can be loaded by other language binding of mxnet.
You also get the benefit being able to directly load/save from cloud storage(S3, HDFS)
Parameters
----------
fname : str
The name of the file.Can be S3 or HDFS address (remember built with S3 support).
Example of fname:
- `s3://my-bucket/path/my-s3-ndarray`
- `hdfs://my-bucket/path/my-hdfs-ndarray`
- `/path-to/my-local-ndarray`
Returns
-------
$out : array ref of NDArrays or hash ref with NDArrays
=cut
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
Save array ref of NDArray or hash of str->NDArray to a binary file.
You can also use Storable to do the job if you only work with Perl.
The advantage of load/save is the file is language agnostic.
This means the file saved using save can be loaded by other language binding of mxnet.
You also get the benefit being able to directly load/save from cloud storage(S3, HDFS)
Parameters
----------
fname : str
The name of the file.Can be S3 or HDFS address (remember built with S3 support).
Example of fname:
- `s3://my-bucket/path/my-s3-ndarray`
- `hdfs://my-bucket/path/my-hdfs-ndarray`
- `/path-to/my-local-ndarray`
$data : array ref of NDArrays or hash ref of NDArrays
The data to be saved.
=cut
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
$filename,
scalar(@$handles),
$handles,
$names
)
);
}
=head2 imdecode
Decode an image from string. Requires OpenCV to work.
Parameters
----------
$str_img : str
binary image data
:$clip_rect : iterable of 4 int
clip decoded image to rectangle (x0, y0, x1, y1)
:$out= : Maybe[NDArray]
output buffer. can be 3 dimensional (c, h, w) or 4 dimensional (n, c, h, w)
:$index : int
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
$channels,
length($str_img),
{ str_img => $str_img, ($out ? (out => $out) : ()) }
);
}
=head2 _new_empty_handle
Returns a new empty handle.
Empty handle can be used to hold result
Returns
-------
a new empty ndarray handle
=cut
sub _new_empty_handle
{
my $hdl = check_call(AI::MXNetCAPI::NDArrayCreateNone());
return $hdl;
}
=head2 _new_alloc_handle
Returns a new handle with specified shape and context.
Empty handle is only used to hold results
Returns
-------
a new empty ndarray handle
=cut
func _new_alloc_handle($shape, $ctx, $delay_alloc, $dtype)
{
my $hdl = check_call(AI::MXNetCAPI::NDArrayCreateEx(
$shape,
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
Wait for all async operations to finish in MXNet.
This function is used for benchmarks only.
=cut
method waitall()
{
check_call(AI::MXNetCAPI::NDArrayWaitAll());
}
=head2 _fresh_grad
Parameters:
----------
Maybe[Bool] $state=
Whether this array's corresponding gradient array
(registered via `autograd->mark_variables`) has been
updated by `autograd->backward` since last reset.
`_fresh_grad` need to be manually set to False
after consuming gradient (usually after updating this
array).
=cut
method _fresh_grad(Maybe[Bool] $state=)
{
if(defined $state)
{
check_call(AI::MXNetCAPI::NDArraySetGradState($self->handle, $state));
return $state;
}
else
{
return scalar(check_call(AI::MXNetCAPI::NDArrayGetGradState($self->handle)));
}
lib/AI/MXNet/NDArray.pm view on Meta::CPAN
[$self->handle],
[defined $out_grad ? $out_grad->handle : undef],
$retain_graph
)
)
}
method CachedOp(@args) { AI::MXNet::CachedOp->new(@args) }
my $lvalue_methods = join "\n", map {"use attributes 'AI::MXNet::NDArray', \\&AI::MXNet::NDArray::$_, 'lvalue';"}
qw/at slice aspdl asmpdl reshape copy sever T astype as_in_context copyto empty zero ones full
array/;
eval << "EOV" if ($^V and $^V >= 5.006007);
{
no warnings qw(misc);
$lvalue_methods
}
EOV
__PACKAGE__->meta->make_immutable;
lib/AI/MXNet/NDArray/Doc.pm view on Meta::CPAN
$key_var_num_args,
$ret_type) = @_;
my $param_str = build_param_doc($arg_names, $arg_types, $arg_desc);
if($key_var_num_args)
{
$desc .= "\nThis function support variable length of positional input."
}
my $doc_str = sprintf("%s\n\n" .
"%s\n" .
"out : NDArray, optional\n" .
" The output NDArray to hold the result.\n\n".
"Returns\n" .
"-------\n" .
"out : NDArray or list of NDArray\n" .
" The output of this function.", $desc, $param_str);
return $doc_str
}
1;
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
=head2 create_optimizer
Create an optimizer with specified name.
Parameters
----------
name: str
Name of required optimizer. Should be the name
of a subclass of Optimizer. Case insensitive.
rescale_grad : float
Rescaling factor on gradient. Normally should be 1/batch_size.
kwargs: dict
Parameters for optimizer
Returns
-------
opt : Optimizer
The result optimizer.
=cut
method create_optimizer(Str $name, %kwargs)
{
if(exists $opt_registry{ lc $name })
{
my $rescale_grad = delete($kwargs{rescale_grad})//1;
return $opt_registry{ lc $name }->new(
rescale_grad => $rescale_grad,
%kwargs
);
}
confess("Cannot find optimizer $name");
}
*create = \&create_optimizer;
has 'rescale_grad' => (is => "rw", isa => "Num", default=>1);
has 'lr' => (is => "rw", isa => "Num");
has 'learning_rate' => (is => "rw", isa => "Num", default => 0.01);
has 'lr_scheduler' => (is => "rw", isa => "Maybe[AI::MXNet::LRScheduler]");
has 'wd' => (is => "rw", isa => "Num", default => 0);
has 'lr_mult' => (is => "rw", isa => "HashRef", default => sub { +{} });
has 'wd_mult' => (is => "rw", isa => "HashRef", , default => sub { +{} });
has 'num_update' => (is => "rw", isa => "Int");
has 'begin_num_update' => (is => "rw", isa => "Int", default => 0);
has '_index_update_count' => (is => "rw", isa => "HashRef", default => sub { +{} });
has 'clip_gradient' => (is => "rw", isa => "Maybe[Num]");
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
----------
learning_rate : float, optional
learning_rate of SGD
momentum : float, optional
momentum value
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
param_idx2name : hash of string/int to float, optional
special treat weight decay in parameter ends with bias, gamma, and beta
multi_precision: bool, optional
Flag to control the internal precision of the optimizer.
False results in using the same precision as the weights (default),
True makes internal 32-bit copy of the weights and applies gradients
in 32-bit precision even if actual weights used in the model have lower precision.
Turning this on can improve convergence and accuracy when training with float16.
=cut
package AI::MXNet::SGD;
use Mouse;
extends 'AI::MXNet::Optimizer';
has 'kwargs' => (is => "rw", isa => "HashRef[Num]");
has 'momentum' => (is => "rw", isa => "Num", default => 0);
has 'multi_precision' => (is => "ro", isa => "Bool", default => 0);
sub BUILD
{
my $self = shift;
$self->kwargs({ rescale_grad => $self->rescale_grad });
if($self->momentum)
{
$self->kwargs->{momentum} = $self->momentum;
}
if($self->clip_gradient)
{
$self->kwargs->{clip_gradient} = $self->clip_gradient;
}
}
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
momentum : float, optional
momentum value
lamda : float, optional
scale DC value
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
param_idx2name : hash ref of string/int to float, optional
special treat weight decay in parameter ends with bias, gamma, and beta
=cut
has 'momentum' => (is => 'ro', isa => 'Num', default => 0);
has 'lamda' => (is => 'ro', isa => 'Num', default => 0.04);
has 'weight_previous' => (is => 'rw', init_arg => undef);
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
Maybe[AI::MXNet::NDArray] $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
my ($mom, $weight_previous) = @{ $state };
if(defined $mom)
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
AI::MXNet::NDArray|Undef $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad = $grad * $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
if($state)
{
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
Stochastic Langevin Dynamics Updater to sample from a distribution.
Parameters
----------
learning_rate : float, optional
learning_rate of SGD
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
param_idx2name : dict of string/int to float, optional
special treat weight decay in parameter ends with bias, gamma, and beta
=cut
package AI::MXNet::SLGD;
use Mouse;
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
AI::MXNet::NDArray|Undef $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
$weight += - $lr/2 * ($grad + $wd * $weight)
+
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
beta2 : float, optional
Exponential decay rate for the second moment estimates.
Default value is set to 0.999.
epsilon : float, optional
Default value is set to 1e-8.
decay_factor : float, optional
Default value is set to 1 - 1e-8.
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
=cut
package AI::MXNet::Adam;
use Mouse;
extends 'AI::MXNet::Optimizer';
has 'kwargs' => (is => "rw", isa => "HashRef[Num]");
has '+learning_rate' => (default => 0.001);
has 'beta1' => (is => "rw", isa => "Num", default => 0.9);
has 'beta2' => (is => "rw", isa => "Num", default => 0.999);
has 'epsilon' => (is => "rw", isa => "Num", default => 1e-8);
has 'decay_factor' => (is => "rw", isa => "Num", default => (1 - 1e-8));
sub BUILD
{
my $self = shift;
$self->kwargs({
rescale_grad => $self->rescale_grad,
beta1 => $self->beta1,
beta2 => $self->beta2,
epsilon => $self->epsilon
});
if($self->clip_gradient)
{
$self->kwargs->{clip_gradient} = $self->clip_gradient;
}
}
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
Parameters
----------
learning_rate : float, optional
Step size.
Default value is set to 0.05.
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
eps: float, optional
A small float number to make the updating processing stable
Default value is set to 1e-7.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
=cut
package AI::MXNet::AdaGrad;
use Mouse;
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
AI::MXNet::NDArray $state
)
{
my $lr = $self->_get_lr($index);
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
my $history = $state;
$history += ($grad * $grad);
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
gamma2: float, optional
"momentum" factor.
Default value if set to 0.9.
Only used if centered=True
epsilon : float, optional
Default value is set to 1e-8.
centered : bool, optional
Use Graves or Tielemans & Hintons version of RMSProp
wd : float, optional
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
clip_weights : float, optional
clip weights in range [-clip_weights, clip_weights]
=cut
package AI::MXNet::RMSProp;
use Mouse;
extends 'AI::MXNet::Optimizer';
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
has 'gamma2' => (is => "ro", isa => "Num", default => 0.9);
has 'epsilon' => (is => "ro", isa => "Num", default => 1e-8);
has 'centered' => (is => "ro", isa => "Bool", default => 0);
has 'clip_weights' => (is => "ro", isa => "Num");
has 'kwargs' => (is => "rw", init_arg => undef);
sub BUILD
{
my $self = shift;
$self->kwargs({
rescale_grad => $self->rescale_grad,
gamma1 => $self->gamma1,
epsilon => $self->epsilon
});
if($self->centered)
{
$self->kwargs->{gamma2} = $self->gamma2;
}
if($self->clip_gradient)
{
$self->kwargs->{clip_gradient} = $self->clip_gradient;
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
http://arxiv.org/abs/1212.5701
Parameters
----------
rho: float
Decay rate for both squared gradients and delta x
epsilon : float
The constant as described in the thesis
wd : float
L2 regularization coefficient add to all the weights
rescale_grad : float, optional
rescaling factor of gradient. Normally should be 1/batch_size.
clip_gradient : float, optional
clip gradient in range [-clip_gradient, clip_gradient]
=cut
package AI::MXNet::AdaDelta;
use Mouse;
extends 'AI::MXNet::Optimizer';
has 'rho' => (is => "rw", isa => "Num", default => 0.9);
has 'epsilon' => (is => "rw", isa => "Num", default => 1e-5);
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
my $wd = $self->_get_wd($index);
$self->_update_count($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
my ($acc_g, $acc_delta) = @{ $state };
$acc_g .= $self->rho * $acc_g + (1 - $self->rho) * $grad * $grad;
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
# Create a state to duplicate weight
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
return AI::MXNet::NDArray->zeros(
$weight->shape,
ctx => $weight->context
);
}
# performs w += rescale_grad * grad
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
AI::MXNet::NDArray $state
)
{
$weight += $grad * $self->rescale_grad;
$state .= $weight;
}
__PACKAGE__->register;
package AI::MXNet::Ftrl;
=head1 NAME
AI::MXNet::Ftrl
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
method update(
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
$self->_update_count($index);
my $wd = $self->_get_wd($index);
my $lr = $self->_get_lr($index);
$grad *= $self->rescale_grad;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
my ($dn, $n) = @{ $state };
$dn += $grad - (($n + $grad * $grad)->sqrt - $n->sqrt) * $weight / $lr;
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
my $wd = $self->_get_wd($index);
my $lr = $self->_get_lr($index);
$self->_update_count($index);
my $t = $self->_index_update_count->{$index};
$lr /= (1 - $self->beta1**$t);
$grad = $grad * $self->rescale_grad + $wd * $weight;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
# update m_t and u_t
lib/AI/MXNet/Optimizer.pm view on Meta::CPAN
Index $index,
AI::MXNet::NDArray $weight,
AI::MXNet::NDArray $grad,
ArrayRef[AI::MXNet::NDArray] $state
)
{
my $wd = $self->_get_wd($index);
my $lr = $self->_get_lr($index);
$self->_update_count($index);
my $t = $self->_index_update_count->{$index};
$grad = $grad * $self->rescale_grad + $wd * $weight;
if($self->clip_gradient)
{
$grad = AI::MXNet::NDArray->clip(
$grad,
-$self->clip_gradient,
$self->clip_gradient
);
}
# warming momentum schedule
my $momentum_t = $self->beta1 * (1 - 0.5 * (0.96**($t * $self->schedule_decay)));
lib/AI/MXNet/RNN.pm view on Meta::CPAN
$period = max(1, $period);
return sub {
my ($iter_no, $sym, $arg, $aux) = @_;
if (($iter_no + 1) % $period == 0)
{
__PACKAGE__->save_rnn_checkpoint($cells, $prefix, $iter_no+1, $sym, $arg, $aux);
}
};
}
## In order to closely resemble the Python's usage
method RNNCell(@args) { AI::MXNet::RNN::Cell->new(@args % 2 ? ('num_hidden', @args) : @args) }
method LSTMCell(@args) { AI::MXNet::RNN::LSTMCell->new(@args % 2 ? ('num_hidden', @args) : @args) }
method GRUCell(@args) { AI::MXNet::RNN::GRUCell->new(@args % 2 ? ('num_hidden', @args) : @args) }
method FusedRNNCell(@args) { AI::MXNet::RNN::FusedCell->new(@args % 2 ? ('num_hidden', @args) : @args) }
method SequentialRNNCell(@args) { AI::MXNet::RNN::SequentialCell->new(@args) }
method BidirectionalCell(@args) { AI::MXNet::RNN::BidirectionalCell->new(@args) }
method DropoutCell(@args) { AI::MXNet::RNN::DropoutCell->new(@args) }
method ZoneoutCell(@args) { AI::MXNet::RNN::ZoneoutCell->new(@args) }
method ConvRNNCell(@args) { AI::MXNet::RNN::ConvCell->new(@args) }
method ConvLSTMCell(@args) { AI::MXNet::RNN::ConvLSTMCell->new(@args) }
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
if(not defined $self->_params)
{
$self->_own_params(1);
$self->_params(AI::MXNet::RNN::Params->new($self->_prefix));
}
else
{
$self->_own_params(0);
}
$self->_modified(0);
$self->reset;
}
=head2 reset
Reset before re-using the cell for another graph
=cut
method reset()
{
$self->_init_counter(-1);
$self->_counter(-1);
}
=head2 call
Construct symbol for one step of RNN.
Parameters
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
method unroll(
Int $length,
Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$inputs=,
Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$begin_state=,
Str :$input_prefix='',
Str :$layout='NTC',
Maybe[Bool] :$merge_outputs=
)
{
$self->reset;
my $axis = index($layout, 'T');
if(not defined $inputs)
{
$inputs = [
map { AI::MXNet::Symbol->Variable("${input_prefix}t${_}_data") } (0..$length-1)
];
}
elsif(blessed($inputs))
{
assert(
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
name => "${name}h2h"
);
my ($i2h_r, $i2h_z);
($i2h_r, $i2h_z, $i2h) = @{ AI::MXNet::Symbol->SliceChannel(
$i2h, num_outputs => 3, name => "${name}_i2h_slice"
) };
my ($h2h_r, $h2h_z);
($h2h_r, $h2h_z, $h2h) = @{ AI::MXNet::Symbol->SliceChannel(
$h2h, num_outputs => 3, name => "${name}_h2h_slice"
) };
my $reset_gate = AI::MXNet::Symbol->Activation(
$i2h_r + $h2h_r, act_type => "sigmoid", name => "${name}_r_act"
);
my $update_gate = AI::MXNet::Symbol->Activation(
$i2h_z + $h2h_z, act_type => "sigmoid", name => "${name}_z_act"
);
my $next_h_tmp = AI::MXNet::Symbol->Activation(
$i2h + $reset_gate * $h2h, act_type => "tanh", name => "${name}_h_act"
);
my $next_h = AI::MXNet::Symbol->_plus(
(1 - $update_gate) * $next_h_tmp, $update_gate * $prev_state_h,
name => "${name}out"
);
return ($next_h, [$next_h]);
}
package AI::MXNet::RNN::FusedCell;
use Mouse;
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
{
for my $direction (@directions)
{
for my $gate (@gate_names)
{
my $name = sprintf('%s%s%d_i2h%s_weight', $self->_prefix, $direction, $layer, $gate);
my $size;
if($layer > 0)
{
$size = $b*$lh*$lh;
$args{$name} = $arr->slice([$p,$p+$size-1])->reshape([$lh, $b*$lh]);
}
else
{
$size = $li*$lh;
$args{$name} = $arr->slice([$p,$p+$size-1])->reshape([$lh, $li]);
}
$p += $size;
}
for my $gate (@gate_names)
{
my $name = sprintf('%s%s%d_h2h%s_weight', $self->_prefix, $direction, $layer, $gate);
my $size = $lh**2;
$args{$name} = $arr->slice([$p,$p+$size-1])->reshape([$lh, $lh]);
$p += $size;
}
}
}
for my $layer (0..$self->_num_layers-1)
{
for my $direction (@directions)
{
for my $gate (@gate_names)
{
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
method unroll(
Int $length,
Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$inputs=,
Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$begin_state=,
Str :$input_prefix='',
Str :$layout='NTC',
Maybe[Bool] :$merge_outputs=
)
{
$self->reset;
my $axis = index($layout, 'T');
$inputs //= AI::MXNet::Symbol->Variable("${input_prefix}data");
if(blessed($inputs))
{
assert(
(@{ $inputs->list_outputs() } == 1),
"unroll doesn't allow grouped symbol as input. Please "
."convert to list first or let unroll handle slicing"
);
if($axis == 1)
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
}
method call(AI::MXNet::Symbol $inputs, AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol] $states)
{
$self->_counter($self->_counter + 1);
my $name = sprintf('%st%d_', $self->_prefix, $self->_counter);
my ($i2h, $h2h) = $self->_conv_forward($inputs, $states, $name);
my ($i2h_r, $i2h_z, $h2h_r, $h2h_z);
($i2h_r, $i2h_z, $i2h) = @{ AI::MXNet::Symbol->SliceChannel($i2h, num_outputs => 3, name => "${name}_i2h_slice") };
($h2h_r, $h2h_z, $h2h) = @{ AI::MXNet::Symbol->SliceChannel($h2h, num_outputs => 3, name => "${name}_h2h_slice") };
my $reset_gate = AI::MXNet::Symbol->Activation(
$i2h_r + $h2h_r, act_type => "sigmoid",
name => "${name}_r_act"
);
my $update_gate = AI::MXNet::Symbol->Activation(
$i2h_z + $h2h_z, act_type => "sigmoid",
name => "${name}_z_act"
);
my $next_h_tmp = $self->_get_activation($i2h + $reset_gate * $h2h, $self->_activation, name => "${name}_h_act");
my $next_h = AI::MXNet::Symbol->_plus(
(1 - $update_gate) * $next_h_tmp, $update_gate * @{$states}[0],
name => "${name}out"
);
return ($next_h, [$next_h]);
}
package AI::MXNet::RNN::ModifierCell;
use Mouse;
use AI::MXNet::Base;
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
"BidirectionalCell doesn't support zoneout since it doesn't support step. ".
"Please add ZoneoutCell to the cells underneath instead."
);
assert(
(not $self->base_cell->isa('AI::MXNet::RNN::SequentialCell') or not $self->_bidirectional),
"Bidirectional SequentialCell doesn't support zoneout. ".
"Please add ZoneoutCell to the cells underneath instead."
);
}
method reset()
{
$self->SUPER::reset;
$self->prev_output(undef);
}
method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states)
{
my ($cell, $p_outputs, $p_states) = ($self->base_cell, $self->zoneout_outputs, $self->zoneout_states);
my ($next_output, $next_states) = &{$cell}($inputs, $states);
my $mask = sub {
my ($p, $like) = @_;
AI::MXNet::Symbol->Dropout(
lib/AI/MXNet/RNN/Cell.pm view on Meta::CPAN
use AI::MXNet::Base;
extends 'AI::MXNet::RNN::ModifierCell';
=head1 NAME
AI::MXNet::RNN::ResidualCell
=cut
=head1 DESCRIPTION
Adds residual connection as described in Wu et al, 2016
(https://arxiv.org/abs/1609.08144).
Output of the cell is output of the base cell plus input.
=cut
method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states)
{
my $output;
($output, $states) = &{$self->base_cell}($inputs, $states);
$output = AI::MXNet::Symbol->elemwise_add($output, $inputs, name => $output->name.'_plus_residual');
return ($output, $states)
}
method unroll(
Int $length,
Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$inputs=,
Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$begin_state=,
Str :$input_prefix='',
Str :$layout='NTC',
Maybe[Bool] :$merge_outputs=
)
{
$self->reset;
$self->base_cell->_modified(0);
my ($outputs, $states) = $self->base_cell->unroll($length, inputs=>$inputs, begin_state=>$begin_state,
layout=>$layout, merge_outputs=>$merge_outputs);
$self->base_cell->_modified(1);
$merge_outputs //= (blessed($outputs) and $outputs->isa('AI::MXNet::Symbol'));
($inputs) = _normalize_sequence($length, $inputs, $layout, $merge_outputs);
if($merge_outputs)
{
$outputs = AI::MXNet::Symbol->elemwise_add($outputs, $inputs, name => $outputs->name . "_plus_residual");
}
else
{
my @temp;
zip(sub {
my ($output_sym, $input_sym) = @_;
push @temp, AI::MXNet::Symbol->elemwise_add($output_sym, $input_sym,
name=>$output_sym->name."_plus_residual");
}, [@{ $outputs }], [@{ $inputs }]);
$outputs = \@temp;
}
return ($outputs, $states);
}
func _normalize_sequence($length, $inputs, $layout, $merge, $in_layout=)
{
assert((defined $inputs),
"unroll(inputs=>undef) has been deprecated. ".
lib/AI/MXNet/RNN/IO.pm view on Meta::CPAN
:$invalid_label : int, default -1
Index for invalid token, like <end-of-sentence>
:$invalid_key : str, default '\n'
Key for invalid token. Uses '\n' for end
of sentence by default.
:$start_label=0 : int
lowest index.
Returns
-------
$result : array ref of array refs of int
encoded sentences
$vocab : hash ref of str -> int
result vocabulary
=cut
method encode_sentences(
ArrayRef[ArrayRef] $sentences,
Maybe[HashRef] :$vocab=,
Int :$invalid_label=-1,
Str :$invalid_key="\n",
Int :$start_label=0
)
lib/AI/MXNet/RNN/IO.pm view on Meta::CPAN
my $new_vocab;
if(not defined $vocab)
{
$vocab = { $invalid_key => $invalid_label };
$new_vocab = 1;
}
else
{
$new_vocab = 0;
}
my @res;
for my $sent (@{ $sentences })
{
my @coded;
for my $word (@{ $sent })
{
if(not exists $vocab->{ $word })
{
assert($new_vocab, "Unknown token: $word");
if($idx == $invalid_label)
{
$idx += 1;
}
$vocab->{$word} = $idx;
$idx += 1;
}
push @coded, $vocab->{ $word };
}
push @res, \@coded;
}
return (\@res, $vocab);
}
package AI::MXNet::BucketSentenceIter;
=encoding UTF-8
=head1 NAME
AI::MXNet::BucketSentenceIter
=cut
lib/AI/MXNet/RNN/IO.pm view on Meta::CPAN
my $buck_len = $buck->shape->at(-1);
for my $j (0..($buck_len - $self->batch_size))
{
if(not $j%$self->batch_size)
{
push @{ $self->idx }, [$i, $j];
}
}
}, $self->data);
$self->curr_idx(0);
$self->reset;
}
method reset()
{
$self->curr_idx(0);
@{ $self->idx } = shuffle(@{ $self->idx });
$self->nddata([]);
$self->ndlabel([]);
for my $buck (@{ $self->data })
{
$buck = pdl_shuffle($buck);
my $label = $buck->zeros;
$label->slice([0, -2], 'X') .= $buck->slice([1, -1], 'X');
lib/AI/MXNet/Random.pm view on Meta::CPAN
=head1 DESCRIPTION
Handling of randomization in MXNet.
=cut
=head2 seed
Seed the random number generators in mxnet.
This seed will affect behavior of functions in this module,
as well as results from executors that contains Random number
such as Dropout operators.
Parameters
----------
seed_state : int
The random number seed to set to all devices.
Notes
-----
The random number generator of mxnet is by default device specific.
( run in 1.645 second using v1.01-cache-2.11-cpan-49f99fa48dc )