Grid-Request

 view release on metacpan or  search on metacpan

lib/Grid/Request.pm  view on Meta::CPAN

    $logger->debug("In _submit_mw.");
    my ($self, $jt, $cmd) = @_;
    unless (defined $jt && defined $cmd) {
        Grid::Request::InvalidArgumentException->throw("Job template and/or command object are not defined.");
    }

    $logger->debug("Setting the command executable.");
    my ($error, $diagnosis) = drmaa_set_attribute($jt, $DRMAA_REMOTE_COMMAND, $WORKER);
    _throw_drmaa("Could not set command executable.") if $error;

    # Calculate how many workers we need. First, calculate the number of iterations by
    # examining the mw arguments
    my $min_count;
    foreach my $param ($cmd->params()) {
        if ($param->type() ne "PARAM") {
            my $count = $param->count();
            if (! defined $min_count) {
                $min_count = $count;
            } else {
                if (($count > 0) && ($count < $min_count)) {
                    $min_count = $count;
                    $logger->debug("New minimum iteration count of $min_count.");
                }
            }
        }
    }

    # Approach for master/worker (mw) jobs:
    #
    # 1. For each parameter, create an argument that contains the argument type, and a 
    #    list of the values to iterate over
    # 2. Calculate the minimum number of iterations from the parameters. In other words,
    #    if there is a mismatch, then you have to take the lowest number of parameters
    #    so that all parameters have defined siblings.
    # 3. Based on the number of iterations N, launch a number of workers on the grid to
    #    process these where the number is some function of N, f(N).
    # 4. Pass the path of the exe to the worker program, so that it knows what to execute
    #    The worker will know what portion of the work to do by the task id that the DRM
    #    gives it. In SGE, this is done with an environment variable: SGE_TASK_ID.
    # 5. Worker will replace $(Index) and $(Name) placeholders with the iteration number or
    #    or the value itself in the output file, error file, input file, args, etc...
    #
    #  General form:
    #  /path/to/worker <executable> <iterations> <workers>  \
    #                    param:blah_blah_blah               \
    #                    dir:<directory>:blah_blah_blah     \
    #                    file:<file>:blah_blah_blah
    #
    # Example: /path/to/worker /path/to/user/command 1000 5 \
    #            dir:/path/to/user/directory:-d $(Name)     \
    #            file:/path/to/user/file:-arg $(Name)       \
    #            param:-plain_arg
    #
    # We use a helper utility and method to determine how to divide the work.

lib/Grid/Request/JobFormulator.pm  view on Meta::CPAN

    $logger->debug("The number of arguments each invocation of $executable will have: $group_size");

    if ( $group_size > 0 ) {
        # Which worker invocation the assembled invocations are for.
        # This module produces invocations for all the tasks that will be launched
        # so we have to replace all $Index occurrences with the right number. Block size
        # dictates how many invocations per task...
        my $task_id = 1; 
        my $block_index = 1; # Initialize a counter to iterate over the blocksize with.

        $logger->debug("Assembling iterations for task ID $task_id.");
        
        # Length of the various argument arrays should all be the same, so we'll just use
        # the length of the first one.
        my $arg_length = scalar(@{ $arg_group_ref->[0] });
        
        # This loop is to iterate across the argument arrays
        for (my $arg_index = 0; $arg_index < $arg_length; $arg_index++) {
            my @exec = ($executable);

            for (my $group_index = 0; $group_index < $group_size; $group_index++) {

lib/Grid/Request/Param.pm  view on Meta::CPAN

}


=item $obj->value([$value]);

B<Description:> A getter/setter that is used to set and retrieve the "value" of
a Grid::Request parameter. Grid::Request jobs may specified an executable to run
on the grid, and the executable may have or require one or more arguments (such as
those typically specified on the command line). These arguments are modeled with
the Grid::Request::Param module. Simple parameters do not trigger any iteration,
however, parameters of type "ARRAY", "DIR", and "FILE", trigger iterations, and
subsequently grid jobs that perform "parameter sweep" (sometimes referred to as
"array jobs").

B<Parameters:> None.

B<Returns:> If called as a getter (no-arguments), returns the current value
of the parameter. If called as a setter, nothing is returned (undef).

=cut 

lib/Grid/Request/Param.pm  view on Meta::CPAN

    $logger->debug("In key.");
    if (@args) {
        $self->{_key} = $args[0];
    } else {
        return $self->{_key};
    }
}

=item $obj->count();

B<Description:> Retrieves the number of iterations that this parameter will
trigger. For parameters of type "ARRAY", it will be the nubmer of array
elements, for parameters of type "DIR", it will be the number of non-hidden
files in the specified directory; for parameters of type "FILE", it will the
number of lines in the file (with the exception of lines with nothing but
whitespace). 

B<Parameters:> None.

B<Returns:> A non-negative integer scalar.