AI-MXNet
view release on metacpan or search on metacpan
lib/AI/MXNet/Executor/Group.pm view on Meta::CPAN
{
$d_src->copyto($d_dst);
}
}
}
}, $data, $targets, $major_axis);
}
# Load data into sliced arrays
func _load_data($batch, $targets, $major_axis)
{
_load_general($batch->data, $targets, $major_axis);
}
# Load label into sliced arrays
func _load_label($batch, $targets, $major_axis)
{
_load_general($batch->label, $targets, $major_axis);
}
# Merge outputs that live on multiple context into one, so that they look
# like living on one context.
func _merge_multi_context($outputs, $major_axis)
{
my @rets;
zip(sub {
my ($tensors, $axis) = @_;
if($axis >= 0)
{
if(@$tensors == 1)
{
push @rets, $tensors->[0];
}
else
{
my $ctx = $tensors->[0]->context;
push @rets, AI::MXNet::NDArray->concat((map { $_->as_in_context($ctx) } @$tensors), { dim => $axis });
}
}
else
{
# negative axis means the there is no batch_size axis, and all the
# results should be the same on each device. We simply take the
# first one, without checking they are actually the same
push @rets, $tensors->[0];
}
}, $outputs, $major_axis);
return \@rets;
}
## TODO
## this class is here because of https://github.com/gfx/p5-Mouse/pull/67
## once 2.4.7 version of Mouse in Ubuntu for affected Perl version
## these accessors should be merged into main class
package AI::MXNet::DataParallelExecutorGroup::_private;
use Mouse;
has [qw/output_layouts label_layouts arg_names aux_names
batch_size slices execs data_arrays
label_arrays param_arrays grad_arrays aux_arrays
data_layouts shared_data_arrays input_grad_arrays
_default_execs state_arrays/
] => (is => 'rw', init_arg => undef);
package AI::MXNet::DataParallelExecutorGroup;
use Mouse;
use AI::MXNet::Base;
use List::Util qw(sum);
=head1 DESCRIPTION
DataParallelExecutorGroup is a group of executors that lives on a group of devices.
This is a helper class used to implement data parallelization. Each mini-batch will
be split and run on the devices.
Parameters for constructor
----------
symbol : AI::MXNet::Symbol
The common symbolic computation graph for all executors.
contexts : ArrayRef[AI::MXNet::Context]
A array ref of contexts.
workload : ArrayRef[Num]
If not undef, could be an array ref of numbers that specify the workload to be assigned
to different context. Larger number indicate heavier workload.
data_shapes : ArrayRef[NameShape|AI::MXNet::DataDesc]
Should be a array ref of [name, shape] array refs, for the shapes of data. Note the order is
important and should be the same as the order that the `DataIter` provide the data.
label_shapes : Maybe[ArrayRef[NameShape|AI::MXNet::DataDesc]]
Should be a array ref of [$name, $shape] array refs, for the shapes of label. Note the order is
important and should be the same as the order that the `DataIter` provide the label.
param_names : ArrayRef[Str]
A array ref of strings, indicating the names of parameters (e.g. weights, filters, etc.)
in the computation graph.
for_training : Bool
Indicate whether the executors should be bind for training. When not doing training,
the memory for gradients will not be allocated.
inputs_need_grad : Bool
Indicate whether the gradients for the input data should be computed. This is currently
not used. It will be useful for implementing composition of modules.
shared_group : AI::MXNet::DataParallelExecutorGroup
Default is undef. This is used in bucketing. When not undef, it should be a executor
group corresponding to a different bucket. In other words, it will correspond to a different
symbol with the same set of parameters (e.g. unrolled RNNs with different lengths).
In this case the memory regions of the parameters will be shared.
logger : Logger
Default is AI::MXNet::Logging->get_logger.
fixed_param_names: Maybe[ArrayRef[Str]]
Indicate parameters to be fixed during training. Parameters in this array ref will not allocate
space for gradient, nor do gradient calculation.
grad_req : ArrayRef[GradReq]|HashRef[GradReq]|GradReq
Requirement for gradient accumulation. Can be 'write', 'add', or 'null'
(default to 'write').
Can be specified globally (str) or for each argument (array ref, hash ref).
state_names: Maybe[ArrayRef[Str]]
=cut
has 'symbol' => (is => 'ro', isa => 'AI::MXNet::Symbol', required => 1);
has 'contexts' => (is => 'ro', isa => 'ArrayRef[AI::MXNet::Context]', required => 1);
has 'workload' => (is => 'ro', isa => 'ArrayRef[Num]', default => sub { [] });
has 'data_shapes' => (is => 'rw', isa => 'ArrayRef[NameShape|AI::MXNet::DataDesc]', required => 1);
has 'label_shapes' => (is => 'rw', isa => 'Maybe[ArrayRef[NameShape|AI::MXNet::DataDesc]]');
has 'param_names' => (is => 'ro', isa => 'ArrayRef[Str]', required => 1);
has 'for_training' => (is => 'ro', isa => 'Bool', required => 1);
has 'inputs_need_grad' => (is => 'ro', isa => 'Bool', default => 0);
has 'shared_group' => (is => 'ro', isa => 'Maybe[AI::MXNet::DataParallelExecutorGroup]');
has 'logger' => (is => 'ro', default => sub { AI::MXNet::Logging->get_logger });
has 'fixed_param_names' => (is => 'rw', isa => 'Maybe[ArrayRef[Str]]');
has 'state_names' => (is => 'rw', isa => 'Maybe[ArrayRef[Str]]');
has 'grad_req' => (is => 'rw', isa => 'ArrayRef[GradReq]|HashRef[GradReq]|GradReq', default=>'write');
has '_p' => (is => 'rw', init_arg => undef);
sub BUILD
{
my $self = shift;
my $p = AI::MXNet::DataParallelExecutorGroup::_private->new;
$p->arg_names($self->symbol->list_arguments);
$p->aux_names($self->symbol->list_auxiliary_states);
$p->execs([]);
$self->_p($p);
$self->grad_req('null') if not $self->for_training;
$self->fixed_param_names([]) unless defined $self->fixed_param_names;
$self->state_names([]) unless defined $self->state_names;
my $data_shapes = [];
for my $d (@{ $self->data_shapes })
{
$d = AI::MXNet::DataDesc->new(name => $d->[0], shape => $d->[1])
unless blessed $d;
push @{ $data_shapes }, $d;
}
$self->data_shapes($data_shapes);
if(defined $self->label_shapes)
{
my $label_shapes = [];
for my $l (@{ $self->label_shapes })
{
$l = AI::MXNet::DataDesc->new(name => $l->[0], shape => $l->[1])
unless blessed $l;
push @{ $label_shapes }, $l;
}
$self->label_shapes($label_shapes);
}
my %data_names = map { $_->name => 1 } @{ $self->data_shapes };
my %param_names = map { $_ => 1 } @{ $self->param_names };
my %fixed_param_names = map { $_ => 1 } @{ $self->fixed_param_names };
my %grad_req;
if(not ref $self->grad_req)
{
for my $k (@{ $self->_p->arg_names })
{
if(exists $param_names{ $k })
{
$grad_req{$k} = exists $fixed_param_names{ $k } ? 'null' : $self->grad_req;
}
elsif(exists $data_names{ $k })
{
$grad_req{$k} = $self->inputs_need_grad ? $self->grad_req : 'null';
}
else
{
$grad_req{$k} = 'null';
}
}
}
elsif(ref $self->grad_req eq 'ARRAY')
{
@grad_req{ @{ $self->_p->arg_names } } = @{ $self->grad_req };
}
else
{
for my $k (@{ $self->_p->arg_names })
lib/AI/MXNet/Executor/Group.pm view on Meta::CPAN
for my $exec (@{ $self->_p->execs })
{
push @tmp, $exec->grad_arrays->[$arg_names{$name}];
}
push @{ $self->_p->input_grad_arrays }, \@tmp;
}
}
$self->_p->aux_arrays([]);
for my $i (0..@{ $self->_p->aux_names }-1)
{
my @tmp;
for my $exec (@{ $self->_p->execs })
{
push @tmp, $exec->aux_arrays->[$i];
}
push @{ $self->_p->aux_arrays }, \@tmp;
}
}
=head2 bind_exec
Bind executors on their respective devices.
Parameters
----------
$data_shapes : ArrayRef[AI::MXNet::DataDesc]
$label_shapes : Maybe[ArrayRef[AI::MXNet::DataDesc]]
$shared_group : Maybe[AI::MXNet::DataParallelExecutorGroup]
$reshape : Bool
=cut
method bind_exec(
ArrayRef[AI::MXNet::DataDesc] $data_shapes,
Maybe[ArrayRef[AI::MXNet::DataDesc]] $label_shapes=,
Maybe[AI::MXNet::DataParallelExecutorGroup] $shared_group=,
Bool $reshape=0
)
{
assert($reshape or not @{ $self->_p->execs });
$self->_p->batch_size(undef);
# calculate workload and bind executors
$self->_p->data_layouts($self->decide_slices($data_shapes));
# call it to make sure labels has the same batch size as data
if(defined $label_shapes)
{
$self->_p->label_layouts($self->decide_slices($label_shapes));
}
for my $i (0..@{ $self->contexts }-1)
{
my $data_shapes_i = $self->_sliced_shape($data_shapes, $i, $self->_p->data_layouts);
my $label_shapes_i = [];
if(defined $label_shapes)
{
$label_shapes_i = $self->_sliced_shape($label_shapes, $i, $self->_p->label_layouts);
}
if($reshape)
{
my %combined_hash = map { $_->name => $_->shape } (@{ $data_shapes_i }, @{ $label_shapes_i });
$self->_p->execs->[$i] = $self->_p->_default_execs->[$i]->reshape(
\%combined_hash,
allow_up_sizing => 1,
);
}
else
{
push @{ $self->_p->execs }, $self->_bind_ith_exec($i, $data_shapes_i, $label_shapes_i, $shared_group);
}
}
$self->data_shapes($data_shapes);
$self->label_shapes($label_shapes);
$self->_collect_arrays;
}
=head2 reshape
Reshape executors.
Parameters
----------
$data_shapes : ArrayRef[AI::MXNet::DataDesc]
$label_shapes : Maybe[ArrayRef[AI::MXNet::DataDesc]]
=cut
method reshape(
ArrayRef[AI::MXNet::DataDesc] $data_shapes,
Maybe[ArrayRef[AI::MXNet::DataDesc]] $label_shapes=
)
{
return if($data_shapes eq $self->data_shapes and $label_shapes eq $self->label_shapes);
if (not defined $self->_p->_default_execs)
{
$self->_p->_default_execs([@{ $self->_p->execs }]);
}
$self->bind_exec($data_shapes, $label_shapes, undef, 1);
}
=head2 set_params
Assign, i.e. copy parameters to all the executors.
Parameters
----------
$arg_params : HashRef[AI::MXNet::NDArray]
A dictionary of name to AI::MXNet::NDArray parameter mapping.
$aux_params : HashRef[AI::MXNet::NDArray]
A dictionary of name to AI::MXNet::NDArray auxiliary variable mapping.
=cut
method set_params(HashRef[AI::MXNet::NDArray] $arg_params, HashRef[AI::MXNet::NDArray] $aux_params, Bool $allow_extra=0)
{
$_->copy_params_from($arg_params, $aux_params, $allow_extra) for @{ $self->_p->execs };
}
=head2 get_params
Copy data from each executor to arg_params and aux_params.
Parameters
----------
$arg_params : HashRef[AI::MXNet::NDArray]
target parameter arrays
$aux_params : HashRef[AI::MXNet::NDArray]
target aux arrays
Notes
-----
- This function will inplace update the NDArrays in arg_params and aux_params.
=cut
method get_params(HashRef[AI::MXNet::NDArray] $arg_params, HashRef[AI::MXNet::NDArray] $aux_params)
{
my $weight = 0;
zip(sub {
my ($name, $block) = @_;
my $weight = sum(map { $_->copyto(AI::MXNet::Context->cpu) } @{ $block }) / @{ $block };
$weight->astype($arg_params->{$name}->dtype)->copyto($arg_params->{$name});
}, $self->param_names, $self->_p->param_arrays);
zip(sub {
my ($name, $block) = @_;
my $weight = sum(map { $_->copyto(AI::MXNet::Context->cpu) } @{ $block }) / @{ $block };
$weight->astype($aux_params->{$name}->dtype)->copyto($aux_params->{$name});
}, $self->_p->aux_names, $self->_p->aux_arrays);
}
method get_states($merge_multi_context=1)
{
assert((not $merge_multi_context), "merge_multi_context=True is not supported for get_states yet.");
return $self->_p->state_arrays;
}
( run in 0.955 second using v1.01-cache-2.11-cpan-39bf76dae61 )