App-Oozie

 view release on metacpan or  search on metacpan

Changes  view on Meta::CPAN

Revision history for App-Oozie

0.020     2024-06-28 18:33:58+02:00 Europe/Brussels
- Re-release.

0.019     2024-06-23 17:23:52+02:00 Europe/Brussels
- dzil seems to have failed to upload. Retrying.

0.018     2024-06-23 17:15:14+02:00 Europe/Brussels
- Use the new TT having the split and remove our own fork.

0.017     2024-06-07 12:47:11+02:00 Europe/Brussels
- Use the ssl change in Net::Hadoop::WebHDFS::LWP.

0.016     2023-12-05 18:19:59+01:00 Europe/Brussels
- Switch to _ttree_obj() for the template processor.

Changes  view on Meta::CPAN

0.007     2023-09-26 12:32:17+02:00 Europe/Brussels
- Fork ttree as a module (not enabled yet).

0.006     2023-09-08 12:47:12+02:00 Europe/Brussels
- New option template_namenode to giving the ability
  to override the Oozie variable nameNode from cli.

0.005     2023-09-06 20:03:36+02:00 Europe/Brussels
- Depends on a newer Net::Hadoop::WebHDFS::LWP for create() operations.
- Handle the upload gracefully and properly.
- More logging.

0.004     2023-09-05 12:09:01+02:00 Europe/Brussels
- Remove a left over hard coded version number.

0.003     2023-09-04 09:51:18+02:00 Europe/Brussels
- Remove XML::Simple.
- Internal cleanup.

0.002     2023-08-31 15:55:08+02:00 Europe/Brussels

eg/workflows/cpan-sample-workflow/job.properties  view on Meta::CPAN

# Note the jobtracker and namenode specs are handled by the oozie-run script, and
# don't need to be specified here. The name will be taken from the directory name.
# Same goes for the job-xml property

# If you intend to be running not a coordinator but a single workflow, just
# remove the coordinator.xml file

# The general advice: put as many properties in the global section of the
# workflow.xml file, and as little as possible here or in the coordinator.xml
# file. The reason is that code changes in the workflow become visible as soon
# as the file is uploaded in HDFS, whereas properties and coordinator.xml
# changes require the coordinator to be killed and restarted on the oozie
# server.

# Actual definition starts after this line

# You might want to keep this as-is, you don't need to touch these and they need to exists
# queueName=default
queueName=my.queue
oozie.use.system.libpath=true

lib/App/Oozie/Deploy.pm  view on Meta::CPAN

    TERMINAL_INFO_LINE_LEN
    WEBHDFS_CREATE_CHUNK_SIZE
);
use Cwd 'abs_path';
use Moo;
use MooX::Options prefer_commandline => 0,
                  protect_argv       => 0,
                  usage_string       => <<'USAGE',
Usage: %c %o

Deploys workflows to HDFS. Specifying names as final arguments will upload only those
USAGE
;

use App::Oozie::Deploy::Template;
use App::Oozie::Deploy::Validate::Spec;
use App::Oozie::Types::Common qw( IsDir IsFile );
use App::Oozie::Util::Misc qw( resolve_tmp_dir trim_slashes );
use App::Oozie::Constants qw( OOZIE_STATES_RUNNING );

use Carp ();

lib/App/Oozie/Deploy.pm  view on Meta::CPAN

    $self->log_versions if $verbose;

    my($update_coord) = $self->_verify_and_compile_all_workflows( $workflows );

    if (!$self->secure_cluster) {
        # Left in place for historial reasons.
        # All clusters should be under Kerberos.
        # Possible removal in a future version.
        #
        # unsafe, but needed when uploading with mapred's uid or hdfs dfs cannot see the files
        chmod oct( DEFAULT_FILE_MODE ), $config->{base_dest};
    }

    my $success = $self->upload_to_hdfs;

    $self->maybe_update_coordinators( $update_coord ) if @{ $update_coord };

    if ($self->prune) {
        $logger->info( '--prune is set, checking workflow directories for old files' );
        for my $workflow ( @{ $workflows } ) {
            $self->prune_path(
                File::Spec->catdir(
                    $config->{hdfs_dest},
                    basename $workflow

lib/App/Oozie/Deploy.pm  view on Meta::CPAN

            if (@subs) {
                for my $subx ( @subs ) {
                    $logger->debug(
                        sprintf 'I will additionally look for workflows in the following sub folder: %s',
                                $subx,
                    );
                }
            }

            push @secondLevelWorkflows, @subs;
            # When deploying a wf/coord, it makes no sense to upload what we have in parent,
            # otherwise, yes, we'll need to update bundle.xml along with any other file on it
            if (! -f $bundleFileLocationGuess) {
              splice @firstLevelWorkflows, $i, 1;
              $i--;
            }
            else {
                $self->logger->debug( 'We have identified this a a bundle.' );
            }
        }
    }

lib/App/Oozie/Deploy.pm  view on Meta::CPAN

                $deleted_files++;
            } else {
                $self->logger->info( "$file->{pathSuffix} has current files, keeping it" );
            }
        }
    }

    return ($total_files == $deleted_files);
}

sub upload_to_hdfs {
    my $self    = shift;
    my $config  = $self->internal_conf;

    if ( $self->dryrun ) {
        $self->logger->warn(
            sprintf 'Skipping upload to HDFS as dryrun was set. Would have uploaded from %s to %s',
                                $config->{base_dest},
                                $config->{hdfs_dest},
        );
        return 1;
    }

    my $success = $self->_copy_to_hdfs_with_webhdfs($config->{base_dest}, $config->{hdfs_dest});
    return $success;
}

lib/App/Oozie/Deploy.pm  view on Meta::CPAN

=head2 pre_verification

=head2 process_templates

=head2 process_workflow

=head2 prune_path

=head2 run

=head2 upload_to_hdfs

=head2 validate_meta_file

=head2 verify_temp_dir

=head2 write_deployment_meta_file

=head1 Accessors

=head2 Overridable from cli