App-Oozie
view release on metacpan or search on metacpan
lib/App/Oozie/Deploy.pm view on Meta::CPAN
my $verbose = $self->verbose;
my $is_file = IsFile->library->get_type( IsFile );
foreach my $file ( @{ $self->required_tt_files } ) {
my $absolute_path = File::Spec->catfile( $ttlib_base_dir, $file );
if ( $verbose ) {
$logger->debug("Assert file: $absolute_path");
}
# assert_valid() does not display the error message, hence the manual check
my $error = $is_file->validate( $absolute_path ) || next;
$logger->logdie( sprintf 'required_tt_files(): %s', $error );
}
if ( $verbose ) {
$logger->debug( join q{=}, $_, $self->$_ ) for qw(
local_oozie_code_path
ttlib_base_dir
);
}
lib/App/Oozie/Deploy.pm view on Meta::CPAN
my $sv = shift || $logger->logdie( 'Spec validator not specified!' );
my $validation_errors_ref = shift;
my $total_errors_ref = shift;
for my $xml_file ( $sv->local_xml_files ) {
my $parsed = $sv->maybe_parse_xml( $xml_file );
if ( my $error = $parsed->{error} ) {
$logger->fatal(
sprintf q{We can't validate %s since parsing failed: %s},
$parsed->{relative_file_name},
$error,
);
${ $validation_errors_ref }++;
${ $total_errors_ref }++;
next; #we don't even have valid XML file at this point, so just skip it
};
$logger->info( sprintf 'Dumping xml to json within %s', $dump_path );
my $json_filename = File::Spec->catfile(
lib/App/Oozie/Deploy.pm view on Meta::CPAN
my($validation_errors, $total_errors);
my($dest, $cvc) = $self->compile_templates(
$workflow,
\$validation_errors,
\$total_errors,
);
if ( $self->write_ownership_to_workflow_xml ) {
$self->validate_meta_file(
File::Spec->catfile( $workflow, $self->meta->file ),
\$validation_errors,
\$total_errors,
{},
);
}
my $sv = $self->_get_spec_validator( $dest );
$self->__maybe_dump_xml_to_json(
lib/App/Oozie/Deploy.pm view on Meta::CPAN
if ( $validation_errors ) {
$self->logger->error( 'Oozie deployment validation status: !!!!! FAILED !!!!!' );
}
else {
$self->logger->info( 'Oozie deployment validation status: OK' );
}
return $validation_errors, $total_errors, $dest, $cvc;
}
sub validate_meta_file {
my $self = shift;
my $file = shift;
$self->logger->info( sprintf 'Extra validation for %s', $file );
return;
}
sub verify_temp_dir {
my $self = shift;
lib/App/Oozie/Deploy.pm view on Meta::CPAN
=head2 process_templates
=head2 process_workflow
=head2 prune_path
=head2 run
=head2 upload_to_hdfs
=head2 validate_meta_file
=head2 verify_temp_dir
=head2 write_deployment_meta_file
=head1 Accessors
=head2 Overridable from cli
=head3 dump_xml_to_json
lib/App/Oozie/Deploy/Validate/DAG/Workflow.pm view on Meta::CPAN
has _vertex_lookup => (
is => 'ro',
isa => HashRef,
default => sub { {} },
);
sub assert {
my $self = shift;
my $file = shift;
my @errors = $self->validate( $file );
if ( @errors ) {
$self->logger->fatal( 'Some errors were encountered.' );
for my $error (@errors) {
$self->logger->fatal( $error->[0] );
$self->logger->fatal( $error->[1] );
}
die "Errors found, aborting.\n";
}
$self->logger->info( "$file validated OK" );
return;
}
sub validate {
my $self = shift;
my $file = shift || die 'No file was specified!';
state $is_end_or_kill = { map { $_ => 1 } qw( end kill ) };
$self->logger->info( "DAG validation for $file" );
if ( ! -e $file ) {
die sprintf 'File %s does not exist', $file;
}
lib/App/Oozie/Deploy/Validate/DAG/Workflow.pm view on Meta::CPAN
=head2 current_nodes
=head2 current_vertices
=head2 dump_graph
=head2 graph_filename
=head2 node_types
=head2 validate
=head1 Possible Extensions
sub _dump_graphviz {
my $self = shift;
my $g = $self->current_graph || die "current_graph is not set!";
my $file = $self->graph_filename;
require Graph::Writer::GraphViz;
lib/App/Oozie/Deploy/Validate/Meta.pm view on Meta::CPAN
);
sub is_invalid_workflow_meta {
my $self = shift;
my $input = shift;
my $type = WorkflowMeta;
my($reason, $what);
eval {
$what = ref $input ? $input : $self->serializer->decode( $input );
$reason = $type->validate_explain( $what );
1;
} or do {
my $eval_error = $@ || 'Zombie error';
$reason = ref $eval_error ? $eval_error : [ $eval_error ];
};
if ( $reason && $what && $self->verbose ) {
require Data::Dumper;
my $d = Data::Dumper->new([ $what], ['*META_YML'] );
$self->logger->debug( sprintf'Raw decoded content without type checks: %s', $d->Dump );
lib/App/Oozie/Deploy/Validate/Oozie.pm view on Meta::CPAN
is => 'rw',
isa => IsExecutable,
required => 1,
);
has oozie_uri => (
is => 'rw',
required => 1,
);
sub validate {
my $self = shift;
my $oozie_xmlfile = shift || die 'No xml specified!';
if ( ! -e $oozie_xmlfile ) {
die "Not a file: $oozie_xmlfile";
}
my($validation_errors, $total_errors);
$self->logger->info(
sprintf 'Oozie validate for %s',
$oozie_xmlfile,
);
my $oozie_uri = $self->oozie_uri;
my $command = [
$self->oozie_cli,
validate => $oozie_xmlfile,
];
my($ok, $err, $full_buf, $stdout_buff, $stderr_buff);
EXEC_VALIDATE: {
# At least Oozie v4.1 does not seem to support an `-oozie` parameter
# In the validate sub command args, hence the need to set the env var.
#
local $ENV{OOZIE_URL} = $oozie_uri if $oozie_uri;
if ( $self->verbose && $oozie_uri ) {
$self->logger->debug(
sprintf 'Overriding the env var OOZIE_URL for validation only: %s',
$oozie_uri,
);
}
( $ok, $err, $full_buf, $stdout_buff, $stderr_buff ) = IPC::Cmd::run(
command => $command,
lib/App/Oozie/Deploy/Validate/Oozie.pm view on Meta::CPAN
TBD
=head1 NAME
App::Oozie::Deploy::Validate::Oozie - Part of the Oozie Workflow validator kit.
=head1 Methods
=head2 oozie_uri
=head2 validate
=head1 SEE ALSO
L<App::Oozie>.
=head1 AUTHORS
=over 4
=item *
lib/App/Oozie/Deploy/Validate/Spec.pm view on Meta::CPAN
my $total_errors = 0;
my @xml_files = $self->local_xml_files;
$logger->info('Validating xml files within workflow directory.');
for my $xml_file ( @xml_files ) {
# Possible improvements
#
# i) let the XML parser filter snippet documents
# ii) or even validate the basic syntax for such documents
# and skip Oozie schema checks (which will fail)
#
if ( index( $xml_file, 'common_datasets.xml' ) != INDEX_NOT_FOUND ) {
next;
}
my $oozie_cli_validation = 1;
my $parsed = $self->maybe_parse_xml( $xml_file );
my $relative_file_name = $parsed->{relative_file_name};
if ( my $error = $parsed->{error} ) {
$logger->fatal("We can't validate $relative_file_name since parsing failed: ", $error );
$validation_errors++;
$total_errors++;
next; #we don't even have valid XML file at this point, so just skip it
};
my($xml_in, $localname) = @{ $parsed }{qw/ xml_in localname /};
if( $localname eq 'workflow-app' ) {
$logger->info(
sprintf '%s identified as %s.',
lib/App/Oozie/Deploy/Validate/Spec.pm view on Meta::CPAN
spec_queue_is_missing_message
verbose
) ),
)->verify( $xml_in );
$validation_errors += $wf_validation_errors;
$total_errors += $wf_total_errors;
# check the DAG is OK
my $dag = App::Oozie::Deploy::Validate::DAG::Workflow->new;
my @dag_errors = $dag->validate( $xml_file );
if ( @dag_errors ) {
$validation_errors += @dag_errors;
$total_errors += @dag_errors;
my $warn_tmpl = 'DAG validation failed: %s';
for my $tuple ( @dag_errors ) {
my($error, $meaning) = @{ $tuple };
$self->logger->warn( sprintf $warn_tmpl, $error );
$self->logger->warn( sprintf $warn_tmpl, $meaning );
}
}
1;
} or do {
my $eval_error = $@ || 'Zombie error';
$logger->warn(
sprintf 'Unable to validate `%s` as %s. Please consider fixing the error: %s',
$relative_file_name,
$localname,
$eval_error,
);
next;
};
}
elsif ( $localname eq 'coordinator-app' ) {
$logger->info(
sprintf '%s identified as %s.',
lib/App/Oozie/Deploy/Validate/Spec.pm view on Meta::CPAN
verbose => $verbose,
)->verify( $xml_in );
$validation_errors += $coord_validation_errors;
$total_errors += $coord_total_errors;
1;
} or do {
my $eval_error = $@ || 'Zombie error';
$logger->warn(
sprintf 'Unable to validate `%s` as %s. Please consider fixing error: %s',
$relative_file_name,
$localname,
$eval_error,
);
next;
};
}
elsif( $localname eq 'bundle-app' ) {
$logger->info(
sprintf '%s identified as %s.',
lib/App/Oozie/Deploy/Validate/Spec.pm view on Meta::CPAN
verbose => $verbose,
)->verify( $xml_in );
$validation_errors += $bundle_validation_errors;
$total_errors += $bundle_total_errors;
1;
} or do {
my $eval_error = $@ || 'Zombie error';
$logger->warn(
sprintf 'Unable to validate `%s` as %s. Please consider fixing error: %s',
$relative_file_name,
$localname,
$eval_error,
);
next;
};
}
else { # we can't identify it and validate, so just yield a warning
$oozie_cli_validation = 0;
$logger->fatal(
sprintf q{We can't validate `%s` since it doesn't look like either workflow-app, coordinator-app or bundle-app.},
$relative_file_name,
);
$validation_errors++;
$total_errors++;
}
if($oozie_cli_validation) {
my($ooz_validation_errors, $ooz_total_errors) = $ov->validate( $xml_file );
$validation_errors += $ooz_validation_errors;
$total_errors += $ooz_total_errors;
}
}
return $validation_errors, $total_errors;
}
1;
lib/App/Oozie/Deploy/Validate/Spec/Coordinator.pm view on Meta::CPAN
sub {
my($h, $key) = @_;
return if $key ne 'timezone' || uc( $h->{$key} ) eq 'UTC';
push @non_utc_tz, $h->{$key};
return;
},
sub {
my($h, $key) = @_;
return if $key ne 'property';
$self->validate_xml_property(
\$validation_errors,
\$total_errors,
$h->{property},
);
for my $property ( @{ $h->{property} } ) {
my $property_name = defined($property->{name})? $property->{name} : EMPTY_STRING;
#if ($property_name eq 'startDate' || $property_name eq 'endDate') {
if (exists $blacklist{$property_name}) {
push @restricted_properties, $h->{$key};
lib/App/Oozie/Deploy/Validate/Spec/Workflow.pm view on Meta::CPAN
# check if global conf parameter contains mapred or default queue configuration
my @queue_array = $global_prop
? (
map { $_->{value} }
grep { $_->{name} =~ m{ queuename }xms }
@{ $global_prop }
)
: ()
;
$self->validate_xml_property(
\$validation_errors,
\$total_errors,
$prop
);
$self->validate_xml_property(
\$validation_errors,
\$total_errors,
$global_prop,
'global'
);
foreach my $queue_value (@queue_array) {
if ( $queue_value =~ 'default' || $queue_value =~ 'mapred' ) {
$logger->error(
'FIXME !!! mapreduce.job.queuename parameter in workflow.xml is set to default or mapred; you are not allowed to deploy workflows in root.mapred or root.default queue.'
lib/App/Oozie/Role/Validate/XML.pm view on Meta::CPAN
use strict;
use warnings;
our $VERSION = '0.020'; # VERSION
use namespace::autoclean -except => [qw/_options_data _options_config/];
use App::Oozie::Constants qw( EMPTY_STRING );
use Moo::Role;
sub validate_xml_property {
my $self = shift;
my $validation_errors_ref = shift;
my $total_errors_ref = shift;
my $prop = shift || return;
my $type = shift || EMPTY_STRING;
my $logger = $self->logger;
my $verbose = $self->verbose;
$type = "$type " if $type;
lib/App/Oozie/Role/Validate/XML.pm view on Meta::CPAN
=head1 DESCRIPTION
This is a Role to be consumed by Oozie validators.
=head1 NAME
App::Oozie::Role::Validate::XML - Internal role for validators.
=head1 Methods
=head2 validate_xml_property
=head1 SEE ALSO
L<App::Oozie>.
=head1 AUTHORS
=over 4
=item *
lib/App/Oozie/Serializer.pm view on Meta::CPAN
$self->_assert_type( $rv ) if $self->enforce_type;
return $rv;
}
sub _assert_type {
my $self = shift;
my $input = shift || die 'No data specified to enforce a type!';
my $type = $self->enforce_type || return;
my $failed = $type->validate_explain( $input, 'USER_INPUT' ) || return;
my $full_msg = join "\n\n", @{ $failed };
require Data::Dumper;
my $d = Data::Dumper->new( [ $input ], [ '*INPUT' ] );
die sprintf <<'DID_NOT_PASS', $full_msg, $d->Dump;
The data structure does not match the type definition: %s
Input was decoded as (compare to the errors above):
%s
( run in 0.286 second using v1.01-cache-2.11-cpan-4d50c553e7e )