App-Oozie
7 results

 view release on metacpan or  search on metacpan

lib/App/Oozie/Deploy.pm  view on Meta::CPAN 

    $logger->info(
        sprintf '%s Starting deployment in %s%s %s',
                    $log_marker,
                    $self->cluster_name,
                    $verbose ? EMPTY_STRING : '. Enable --verbose to see the underlying commands',
                    $log_marker,
    );

    $self->log_versions if $verbose;

    my($update_coord) = $self->_verify_and_compile_all_workflows( $workflows );

    if (!$self->secure_cluster) {
        # Left in place for historial reasons.
        # All clusters should be under Kerberos.
        # Possible removal in a future version.
        #
        # unsafe, but needed when uploading with mapred's uid or hdfs dfs cannot see the files
        chmod oct( DEFAULT_FILE_MODE ), $config->{base_dest};
    }

lib/App/Oozie/Deploy.pm  view on Meta::CPAN 

        sprintf '%s Completed successfully in %s (took %s) %s',
                    $log_marker,
                    sprintf( '%s%s', $self->cluster_name, ( $dryrun ? ' (dryrun is set)' : EMPTY_STRING ) ),
                    duration_exact( time - $run_start_epoch ),
                    $log_marker,
    );

    return $success;
}

sub _verify_and_compile_all_workflows {
    my $self = shift;
    my $workflows = shift;

    my $logger    = $self->logger;

    if ( ! is_arrayref $workflows || ! @{ $workflows } ) {
        $logger->logdie( 'Please give one or several workflow name(s) on the command line (glob pattern accepted). Also see --help' );
    }

    $self->pre_verification( $workflows );
    $self->verify_temp_dir;

    if (   $self->gitfeatures
        && ! $self->gitforce
    ) {
        $self->verify_git_tag;
    }

    my $wfs = $self->collect_names_to_deploy( $workflows );
    my($total_errors, $validation_errors);

    my @update_coord;
    for my $workflow ( @{ $wfs } ) {
        my($t_validation_errors, $t_total_errors, $dest, $cvc) =  $self->process_workflow( $workflow );
        push @update_coord, $self->guess_running_coordinator( $workflow, $cvc, $dest );
        $total_errors      += $t_validation_errors;

lib/App/Oozie/Deploy.pm  view on Meta::CPAN 

    }

    my $sv = $self->_get_spec_validator( $dest );

    $self->__maybe_dump_xml_to_json(
        $sv,
        \$validation_errors,
        \$total_errors,
    ) if $self->dump_xml_to_json;

    my($spec_validation_errors, $spec_total_errors) = $sv->verify( $workflow );
    $validation_errors += $spec_validation_errors;
    $total_errors      += $spec_total_errors;

    $self->create_deployment_meta_file( $dest, $workflow, $total_errors );

    if ( $validation_errors ) {
        $self->logger->error( 'Oozie deployment validation status: !!!!! FAILED !!!!!' );
    }
    else {
        $self->logger->info( 'Oozie deployment validation status: OK' );

lib/App/Oozie/Deploy.pm  view on Meta::CPAN 


sub validate_meta_file {
    my $self = shift;
    my $file = shift;

    $self->logger->info( sprintf 'Extra validation for %s', $file );

    return;
}

sub verify_temp_dir {
    my $self         = shift;
    my $user_setting = $ENV{TMPDIR} || return;
    my $logger       = $self->logger;

    # The path needs to be readable by mapred in order the deploy to be successful
    # Some users have this set in their environment to paths lacking relevant
    # permissions leading to failures.
    #
    # If the path is bogus, then by removing the setting locally in here
    # will lead the temporary directory to be created inside "/tmp" by default.

lib/App/Oozie/Deploy.pm  view on Meta::CPAN 

=head2 process_workflow

=head2 prune_path

=head2 run

=head2 upload_to_hdfs

=head2 validate_meta_file

=head2 verify_temp_dir

=head2 write_deployment_meta_file

=head1 Accessors

=head2 Overridable from cli

=head3 dump_xml_to_json

=head3 hdfs_dest

lib/App/Oozie/Deploy/Validate/Spec.pm  view on Meta::CPAN 

    my $self = shift;
    my $dest = $self->local_path;
    return File::Find::Rule
            ->file
            ->maxdepth( 1       )
            ->name(     '*.xml' )
            ->in(       $dest   )
    ;
}

sub verify {
    my $self    = shift;
    my $dest    = $self->local_path;
    my $verbose = $self->verbose;
    my $logger  = $self->logger;

    my $ov = App::Oozie::Deploy::Validate::Oozie->new(
                map { $_ => $self->$_ }
                    qw(
                        oozie_cli
                        oozie_uri

lib/App/Oozie/Deploy/Validate/Spec.pm  view on Meta::CPAN 

                    $wf_total_errors,
                ) = App::Oozie::Deploy::Validate::Spec::Workflow->new(
                        file => $xml_file,
                        ( map { $_ => $self->$_ } qw(
                            email_validator
                            max_node_name_len
                            max_wf_xml_length
                            spec_queue_is_missing_message
                            verbose
                        ) ),
                    )->verify( $xml_in );

                $validation_errors += $wf_validation_errors;
                $total_errors      += $wf_total_errors;

                # check the DAG is OK
                my $dag = App::Oozie::Deploy::Validate::DAG::Workflow->new;
                my @dag_errors = $dag->validate( $xml_file );

                if ( @dag_errors ) {
                    $validation_errors += @dag_errors;

lib/App/Oozie/Deploy/Validate/Spec.pm  view on Meta::CPAN 

            $logger->info(
                sprintf '%s identified as %s.',
                        $relative_file_name,
                        $localname,
            );
            eval {
                my ($coord_validation_errors,
                    $coord_total_errors,
                ) = App::Oozie::Deploy::Validate::Spec::Coordinator->new(
                    verbose => $verbose,
                )->verify( $xml_in );

                $validation_errors += $coord_validation_errors;
                $total_errors      += $coord_total_errors;

                1;
            } or do {
                my $eval_error = $@ || 'Zombie error';
                $logger->warn(
                    sprintf 'Unable to validate `%s` as %s. Please consider fixing error: %s',
                                $relative_file_name,

lib/App/Oozie/Deploy/Validate/Spec.pm  view on Meta::CPAN 

            $logger->info(
                sprintf '%s identified as %s.',
                        $relative_file_name,
                        $localname,
            );
            eval {
                my ($bundle_validation_errors,
                    $bundle_total_errors,
                ) = App::Oozie::Deploy::Validate::Spec::Bundle->new(
                    verbose => $verbose,
                )->verify( $xml_in );

                $validation_errors += $bundle_validation_errors;
                $total_errors      += $bundle_total_errors;

                1;
            } or do {
                my $eval_error = $@ || 'Zombie error';
                $logger->warn(
                    sprintf 'Unable to validate `%s` as %s. Please consider fixing error: %s',
                                $relative_file_name,

lib/App/Oozie/Deploy/Validate/Spec.pm  view on Meta::CPAN 

=head2 max_wf_xml_length

=head2 maybe_parse_xml

=head2 oozie_client_jar

=head2 oozie_uri

=head2 spec_queue_is_missing_message

=head2 verify

=head1 SEE ALSO

L<App::Oozie>.

=head1 AUTHORS

=over 4

=item *

lib/App/Oozie/Deploy/Validate/Spec/Bundle.pm  view on Meta::CPAN 

use Moo;
use MooX::Options;
use List::MoreUtils qw( uniq );
use App::Oozie::Types::Common qw( IsFile );

with qw(
    App::Oozie::Role::Log
    App::Oozie::Role::Fields::Generic
);

sub verify {
    my $self   = shift;
    my $xml_in = shift;

    my($validation_errors, $total_errors);
    my $looper;
    $looper = sub  {
        my $hash = shift;
        my $cb   = shift;
        foreach my $key ( keys %{ $hash } ) {
            $cb->( $hash, $key );

lib/App/Oozie/Deploy/Validate/Spec/Bundle.pm  view on Meta::CPAN 

=head1 DESCRIPTION

TBD

=head1 NAME

App::Oozie::Deploy::Validate::Spec::Bundle - Part of the Oozie Workflow validator kit.

=head1 Methods

=head2 verify

=head1 SEE ALSO

L<App::Oozie>.

=head1 AUTHORS

=over 4

=item *

lib/App/Oozie/Deploy/Validate/Spec/Coordinator.pm  view on Meta::CPAN 

use List::MoreUtils qw( uniq );
use App::Oozie::Types::Common qw( IsFile );
use App::Oozie::Constants qw( EMPTY_STRING );

with qw(
    App::Oozie::Role::Log
    App::Oozie::Role::Fields::Generic
    App::Oozie::Role::Validate::XML
);

sub verify {
    my $self   = shift;
    my $xml_in = shift;

    my($validation_errors, $total_errors);
    my $looper;

    $looper = sub  {
        my $hash = shift;
        my $non_utc = shift;
        my $wrong_parameters = shift;

lib/App/Oozie/Deploy/Validate/Spec/Coordinator.pm  view on Meta::CPAN 

=head1 DESCRIPTION

TBD

=head1 NAME

App::Oozie::Deploy::Validate::Spec::Coordinator - Part of the Oozie Workflow validator kit.

=head1 Methods

=head2 verify

=head1 SEE ALSO

L<App::Oozie>.

=head1 AUTHORS

=over 4

=item *

lib/App/Oozie/Deploy/Validate/Spec/Workflow.pm  view on Meta::CPAN 

      </property>
    [% PROCESS workflow_global_xml_end %]

The [% ... %] tags are probably already in your
workflow.xml.

NO_QUEUE_MSG
    },
);

sub verify {
    my $self   = shift;
    my $xml_in = shift;

    my $file              = $self->file;
    my $wf_size           = $self->file_size;
    my $max_wf_xml_length = $self->max_wf_xml_length;
    my $max_node_name_len = $self->max_node_name_len;

    my $logger = $self->logger;

lib/App/Oozie/Deploy/Validate/Spec/Workflow.pm  view on Meta::CPAN 


Plese rename it as your job will fail eventually at run time.

LONG_ACTION_NAME
            $logger->warn( $msg );
            $validation_errors++;
            $total_errors++;
        }
        my($action_validation_errors,
           $action_total_errors
        ) = $self->verify_queue_name( $action, $global_prop );
        $validation_errors += $action_validation_errors;
        $total_errors      += $action_total_errors;
    }

    return $validation_errors // 0, $total_errors // 0;
}

sub verify_queue_name {
    my($self, $action, $global_prop) = @_;

    my $logger = $self->logger;
    $logger->info( sprintf 'Verifying %s', $self->queue_conf_key_name );

    # check if workflow has defined queuname globally
    my $needs_verification =   ! $global_prop
                            || ! exists $global_prop->{ $self->queue_conf_key_name }
    ;

lib/App/Oozie/Deploy/Validate/Spec/Workflow.pm  view on Meta::CPAN 

=head2 file_size

=head2 max_node_name_len

=head2 max_wf_xml_length

=head2 queue_conf_key_name

=head2 spec_queue_is_missing_message

=head2 verify

=head2 verify_queue_name

=head1 SEE ALSO

L<App::Oozie>.

=head1 AUTHORS

=over 4

=item *

lib/App/Oozie/Role/Git.pm  view on Meta::CPAN 

    } or do {
        my $eval_error = $@ || 'Zombie error';
        $rv = sprintf 'Failed to get modification date. %s;',
                        $eval_error,
            ;
    };

    return $rv;
}

sub verify_git_tag {
    my $self = shift;

    die 'Git features are disabled!' if ! $self->gitfeatures;

    my $oozie_base_dir = $self->local_oozie_code_path;
    my $repo_dir       = $self->git_repo_path;
    my $logger         = $self->logger;

    # If you are using http://git-deploy.github.io/ (for example)
    # then you may have tagged releases for live code which you
    # can have a check against in here.
    my $git_tag        = $self->git_tag_fetcher->() || die 'Failed to locate a git tag!';
    my $gitforce       = $self->gitforce;

    my $git = $self->git;

    my $verify;
    eval {
        $verify = $git->run(
                    'show-ref',
                        '--tags',
                        '--verify',
                        'refs/tags/' . $git_tag,
                );
        1;
    } or do {
        my $eval_error = $@ || 'Zombie error';
        if ( $eval_error =~ m{ fatal: .* \Qnot a valid ref\E  }xms ) {
            $logger->error(
                'Your repo does not seem to have the current live git-deploy tag.',
                q{Be sure that your repo is up to date if you're using a local copy!},
                'If you are in a branch, be sure to have the latest changes (merge/rebase).'

lib/App/Oozie/Role/Git.pm  view on Meta::CPAN 

=head2 get_git_info_on_all_files_in

=head2 get_git_sha1_of_folder

=head2 get_git_status

=head2 get_latest_git_commit

=head2 get_latest_git_tag

=head2 verify_git_tag

=head1 Accessors

=head2 Overridable from cli

=head3 git_repo_path

=head3 gitfeatures

=head3 gitforce

lib/App/Oozie/Run.pm  view on Meta::CPAN 

    my $logger = $self->logger;

    my @extra_oozie_args;

    my @define = @{ $self->define };

    my %extra_def = ();
    # We are not supporting sla for bundles (yet)
    if ( !($self->type eq 'bundle') ) {
      %extra_def = (
          $self->verify_sla,
      );
    }

    if ( $self->type eq 'wf' ) {
        %extra_def = (
            %extra_def,
            $self->check_coordinator_function_calls({
                map { (split m{ [=] }xms, $_)[0] => 1 } @define
            }),
        );

lib/App/Oozie/Run.pm  view on Meta::CPAN 

        oozie_uri     => $self->oozie_uri,
        start_time    => $start_time,
        type          => $self->type,
        workflow_path => $self->path . ($self->type eq 'bundle'? '/bundle.xml' : EMPTY_STRING) ,
        path          => $nameNode . $self->path,
    );

    return \@cmd_tmpl, \%cmd_param;
}

sub verify_sla {
    my $self = shift;
    my %rv;

    # check the SLA parameter is provided if the workflow has an SLA block
    eval {
        my $raw  = $self->hdfs->read(
                        File::Spec->catfile( $self->path, 'workflow.xml' )
                    );
        if ( $raw =~ m{ sla[:]info }xms ) {
            if ( ! $self->sla_duration ) {

lib/App/Oozie/Run.pm  view on Meta::CPAN 

=head3 execute

=head3 log_console_url

=head3 probe_settings

=head3 run

=head3 setup_dates

=head3 verify_sla

=head1 Accessors

=head2 Overridable from cli

=head3 appname

=head3 dates_from_properties

=head3 define



( run in 0.973 second using v1.01-cache-2.11-cpan-e1769b4cff6 )